refactor: change custom logic for context in worker pool

pkg/platform.go

@@ -1,6 +1,7 @@
 package synchronizator
 
 import (
+	"context"
 	"fmt"
 	"slices"
 	"time"
@@ -15,7 +16,7 @@ type Platform struct {
 
 func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error {
 	fetchWrapper := func(offset int) ([]*Collection, error) {
-		fmt.Printf("Requesting offset: %v\n", offset)
+		// fmt.Printf("Requesting offset: %v\n", offset)
 
 		if offset == 10 {
 			return nil, fmt.Errorf("Simulated error jeje")
@@ -32,30 +33,51 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
 		return collections, nil
 	}
 
-	// 5 request per minute
-	rate_limit := NewRateLimit(5, time.Minute)
+	ctx, cancel := context.WithCancel(context.Background())
 
-	manager := createWorkerPool[int, []*Collection](5, 2, rate_limit, fetchWrapper)
+	config := &WorkConfig{
+		max_workers:     5,
+		max_retries:     2,
+		base_retry_time: time.Second,
+		rate_limit:      NewRateLimit(5, time.Minute),
+	}
+
+	tasks := make(chan int)
 
 	// TODO: get number of page dynamically and change Fetcher signature
 	pages := 4
 
+	results, errors, done := asyncTaskRunner[int, []*Collection](ctx, tasks, config, fetchWrapper)
+
 	for i := range pages {
-		err := manager.AddWork(i * start_pagination.Limit)
-		if err != nil {
-			return err
+		tasks <- i * start_pagination.Limit
+	}
+
+	close(tasks)
+
+loop:
+	for {
+		select {
+		case collection, ok := <-results:
+			if !ok {
+				continue
+			}
+			platform.Collections = slices.Concat(platform.Collections, collection)
+		case error, ok := <-errors:
+			if !ok {
+				continue
+			}
+
+			fmt.Printf("There was an error: %v\n", error)
+			cancel()
+		case <-ctx.Done():
+			break loop
+		case <-done:
+			break loop
 		}
 	}
 
-	successUnits := 0
-	for collections := range manager.GetWorkUnit() {
-		platform.Collections = slices.Concat(platform.Collections, collections)
-		successUnits++
-	}
-
-	if successUnits != pages {
-		return fmt.Errorf("Units failed: %v", manager.GetFailedUnits())
-	}
+	fmt.Printf("Collections: %v\n", len(platform.Collections))
 
 	err := BulkCreateNode(platform._conn, platform.Collections)
 	if err != nil {