generated from alecodes/base-template
feat: add rate limit to worker pool
This commit is contained in:
parent
30cf6dfff2
commit
2d1041f167
2 changed files with 34 additions and 3 deletions
|
|
@ -3,6 +3,7 @@ package synchronizator
|
|||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Utility struct to represent a collection of nodes, it's a [Node] itself so all
|
||||
|
|
@ -27,7 +28,10 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
|
|||
return collections, nil
|
||||
}
|
||||
|
||||
manager := createWorkerPool[int, []*Collection](5, fetchWrapper)
|
||||
// 5 request per minute
|
||||
rate_limit := NewRateLimit(5, time.Minute)
|
||||
|
||||
manager := createWorkerPool[int, []*Collection](5, rate_limit, fetchWrapper)
|
||||
|
||||
// TODO: get number of page dynamically and change Fetcher signature
|
||||
pages := 4
|
||||
|
|
|
|||
Reference in a new issue