feat: add rate limit to worker pool

This commit is contained in:
Alexander Navarro 2024-11-27 13:46:42 -03:00
parent 30cf6dfff2
commit 2d1041f167
2 changed files with 34 additions and 3 deletions

View file

@ -3,6 +3,7 @@ package synchronizator
import (
"fmt"
"slices"
"time"
)
// Utility struct to represent a collection of nodes, it's a [Node] itself so all
@ -27,7 +28,10 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
return collections, nil
}
manager := createWorkerPool[int, []*Collection](5, fetchWrapper)
// 5 request per minute
rate_limit := NewRateLimit(5, time.Minute)
manager := createWorkerPool[int, []*Collection](5, rate_limit, fetchWrapper)
// TODO: get number of page dynamically and change Fetcher signature
pages := 4