alecodes/synchronizator-go
Archived
2
0
Fork 0
generated from alecodes/base-template
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

feat: add basic retry capabilities to worker pool

Browse source
This commit is contained in:
Alexander Navarro 2024-11-28 20:48:56 -03:00
parent 2d1041f167
commit b342437a43
2 changed files with 123 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

12
pkg/platform.go
View file

@ -17,6 +17,10 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
fetchWrapper := func(offset int) ([]*Collection, error) {
fmt.Printf("Requesting offset: %v\n", offset)
if offset == 10 {
return nil, fmt.Errorf("Simulated error jeje")
}
pagination := start_pagination
pagination.Offset = offset
@ -31,7 +35,7 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
// 5 request per minute
rate_limit := NewRateLimit(5, time.Minute)
manager := createWorkerPool[int, []*Collection](5, rate_limit, fetchWrapper)
manager := createWorkerPool[int, []*Collection](5, 2, rate_limit, fetchWrapper)
// TODO: get number of page dynamically and change Fetcher signature
pages := 4
@ -43,8 +47,14 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
}
}
successUnits := 0
for collections := range manager.GetWorkUnit() {
platform.Collections = slices.Concat(platform.Collections, collections)
successUnits++
}
if successUnits != pages {
return fmt.Errorf("Units failed: %v", manager.GetFailedUnits())
}
err := BulkCreateNode(platform._conn, platform.Collections)