Go High-Throughput Worker Pool with Context Cancellation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
package pool

import (
	"context"
	"sync"
)

type Job func(ctx context.Context) error

func RunWorkerPool(ctx context.Context, workers int, jobs <-chan Job) <-chan error {
	outErrors := make(chan error, workers)
	var wg sync.WaitGroup

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ctx.Done():
					return
				case job, ok := <-jobs:
					if !ok {
						return
					}
					if err := job(ctx); err != nil {
						select {
						case outErrors <- err:
						case <-ctx.Done():
							return
						}
					}
				}
			}
		}()
	}

	go func() {
		wg.Wait()
		close(outErrors)
	}()

	return outErrors
}