feat: add basic worker pool

pkg/platform.go

@@ -1,6 +1,9 @@
 package synchronizator
 
-import "slices"
+import (
+	"fmt"
+	"slices"
+)
 
 // Utility struct to represent a collection of nodes, it's a [Node] itself so all
 // the node's functionality is available.
@@ -10,17 +13,37 @@ type Platform struct {
 }
 
 func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error {
-	collections, pagination, err := fetcher(start_pagination)
-	if err != nil {
-		return err
-	}
-	platform.Collections = slices.Concat(platform.Collections, collections)
+	fetchWrapper := func(offset int) ([]*Collection, error) {
+		fmt.Printf("Requesting offset: %v\n", offset)
 
-	if pagination.HasMore {
-		return platform.FetchCollections(fetcher, pagination)
+		pagination := start_pagination
+
+		pagination.Offset = offset
+		collections, _, err := fetcher(pagination)
+		if err != nil {
+			return nil, err
+		}
+
+		return collections, nil
 	}
 
-	err = BulkCreateNode(platform._conn, platform.Collections)
+	manager := createWorkerPool[int, []*Collection](5, fetchWrapper)
+
+	// TODO: get number of page dynamically and change Fetcher signature
+	pages := 4
+
+	for i := range pages {
+		err := manager.AddWork(i * start_pagination.Limit)
+		if err != nil {
+			return err
+		}
+	}
+
+	for collections := range manager.GetWorkUnit() {
+		platform.Collections = slices.Concat(platform.Collections, collections)
+	}
+
+	err := BulkCreateNode(platform._conn, platform.Collections)
 	if err != nil {
 		return err
 	}