1 Star 0 Fork 0

zhangjungang/beats

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
queuetest.go 6.88 KB
一键复制 编辑 原始数据 按行查看 历史
Steffen Siering 提交于 2017-07-31 22:21 . Pipeline cleanups (#4776)
package queuetest
import (
"sync"
"testing"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/publisher/queue"
)
type QueueFactory func() queue.Queue
type workerFactory func(*sync.WaitGroup, interface{}, *TestLogger, queue.Queue) func()
func TestSingleProducerConsumer(
t *testing.T,
events, batchSize int,
factory QueueFactory,
) {
tests := []struct {
name string
producers, consumers workerFactory
}{
{
"single producer, consumer without ack, complete batches",
makeProducer(events, false, countEvent),
makeConsumer(events, -1),
},
{
"single producer, consumer, without ack, limited batches",
makeProducer(events, false, countEvent),
makeConsumer(events, batchSize),
},
{
"single producer, consumer, with ack, complete batches",
makeProducer(events, true, countEvent),
makeConsumer(events, -1),
},
{
"single producer, consumer, with ack, limited batches",
makeProducer(events, true, countEvent),
makeConsumer(events, batchSize),
},
}
for _, test := range tests {
t.Run(test.name, withLogOutput(func(t *testing.T) {
log := NewTestLogger(t)
log.Debug("run test: ", test.name)
queue := factory()
defer func() {
err := queue.Close()
if err != nil {
t.Error(err)
}
}()
var wg sync.WaitGroup
go test.producers(&wg, nil, log, queue)()
go test.consumers(&wg, nil, log, queue)()
wg.Wait()
}))
}
}
func TestMultiProducerConsumer(
t *testing.T,
events, batchSize int,
factory QueueFactory,
) {
tests := []struct {
name string
producers, consumers workerFactory
}{
{
"2 producers, 1 consumer, without ack, complete batches",
multiple(
makeProducer(events, false, countEvent),
makeProducer(events, false, countEvent),
),
makeConsumer(events*2, -1),
},
{
"2 producers, 1 consumer, all ack, complete batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, true, countEvent),
),
makeConsumer(events*2, -1),
},
{
"2 producers, 1 consumer, 1 ack, complete batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, false, countEvent),
),
makeConsumer(events*2, -1),
},
{
"2 producers, 1 consumer, without ack, limited batches",
multiple(
makeProducer(events, false, countEvent),
makeProducer(events, false, countEvent),
),
makeConsumer(events*2, batchSize),
},
{
"2 producers, 1 consumer, all ack, limited batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, true, countEvent),
),
makeConsumer(events*2, batchSize),
},
{
"2 producers, 1 consumer, 1 ack, limited batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, false, countEvent),
),
makeConsumer(events*2, batchSize),
},
{
"1 producer, 2 consumers, without ack, complete batches",
makeProducer(events, true, countEvent),
multiConsumer(2, events, -1),
},
{
"1 producer, 2 consumers, without ack, limited batches",
makeProducer(events, true, countEvent),
multiConsumer(2, events, -1),
},
{
"2 producers, 2 consumer, without ack, complete batches",
multiple(
makeProducer(events, false, countEvent),
makeProducer(events, false, countEvent),
),
multiConsumer(2, events*2, -1),
},
{
"2 producers, 2 consumer, all ack, complete batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, true, countEvent),
),
multiConsumer(2, events*2, -1),
},
{
"2 producers, 2 consumer, 1 ack, complete batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, false, countEvent),
),
multiConsumer(2, events*2, -1),
},
{
"2 producers, 2 consumer, without ack, limited batches",
multiple(
makeProducer(events, false, countEvent),
makeProducer(events, false, countEvent),
),
multiConsumer(2, events*2, batchSize),
},
{
"2 producers, 2 consumer, all ack, limited batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, true, countEvent),
),
multiConsumer(2, events*2, batchSize),
},
{
"2 producers, 2 consumer, 1 ack, limited batches",
multiple(
makeProducer(events, true, countEvent),
makeProducer(events, false, countEvent),
),
multiConsumer(2, events*2, batchSize),
},
}
for _, test := range tests {
t.Run(test.name, withLogOutput(func(t *testing.T) {
log := NewTestLogger(t)
log.Debug("run test: ", test.name)
queue := factory()
defer func() {
err := queue.Close()
if err != nil {
t.Error(err)
}
}()
var wg sync.WaitGroup
go test.producers(&wg, nil, log, queue)()
go test.consumers(&wg, nil, log, queue)()
wg.Wait()
}))
}
}
func multiple(
fns ...workerFactory,
) workerFactory {
return func(wg *sync.WaitGroup, info interface{}, log *TestLogger, queue queue.Queue) func() {
runners := make([]func(), len(fns))
for i, gen := range fns {
runners[i] = gen(wg, info, log, queue)
}
return func() {
for _, r := range runners {
go r()
}
}
}
}
func makeProducer(
maxEvents int,
waitACK bool,
makeFields func(int) common.MapStr,
) func(*sync.WaitGroup, interface{}, *TestLogger, queue.Queue) func() {
return func(wg *sync.WaitGroup, info interface{}, log *TestLogger, b queue.Queue) func() {
wg.Add(1)
return func() {
defer wg.Done()
log.Debug("start producer")
defer log.Debug("stop producer")
var (
ackWG sync.WaitGroup
ackCB func(int)
)
if waitACK {
ackWG.Add(maxEvents)
total := 0
ackCB = func(N int) {
total += N
log.Debugf("producer ACK: N=%v, total=%v\n", N, total)
for i := 0; i < N; i++ {
ackWG.Done()
}
}
}
producer := b.Producer(queue.ProducerConfig{
ACK: ackCB,
})
for i := 0; i < maxEvents; i++ {
producer.Publish(makeEvent(makeFields(i)))
}
ackWG.Wait()
}
}
}
func makeConsumer(maxEvents, batchSize int) workerFactory {
return multiConsumer(1, maxEvents, batchSize)
}
func multiConsumer(numConsumers, maxEvents, batchSize int) workerFactory {
return func(wg *sync.WaitGroup, info interface{}, log *TestLogger, b queue.Queue) func() {
wg.Add(1)
return func() {
defer wg.Done()
var events sync.WaitGroup
consumers := make([]queue.Consumer, numConsumers)
for i := range consumers {
consumers[i] = b.Consumer()
}
events.Add(maxEvents)
for _, c := range consumers {
c := c
wg.Add(1)
go func() {
defer wg.Done()
for {
batch, err := c.Get(batchSize)
if err != nil {
return
}
for range batch.Events() {
events.Done()
}
batch.ACK()
}
}()
}
events.Wait()
// disconnect consumers
for _, c := range consumers {
c.Close()
}
}
}
}
func countEvent(i int) common.MapStr {
return common.MapStr{
"count": i,
}
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/zhangjungang/beats.git
git@gitee.com:zhangjungang/beats.git
zhangjungang
beats
beats
v6.1.1

搜索帮助