feat: auto calculate pagination in fetcher

also refactor a little the worker pool

pkg/platform.go

@@ -14,25 +14,12 @@ type Platform struct {
 	Collections []*Collection // Child nodes
 }
 
-func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error {
-	fetchWrapper := func(offset int) ([]*Collection, error) {
-		// fmt.Printf("Requesting offset: %v\n", offset)
-
-		if offset == 10 {
-			time.Sleep(time.Second * 5)
-		}
-
-		pagination := start_pagination
-
-		pagination.Offset = offset
-		collections, _, err := fetcher(pagination)
-		if err != nil {
-			return nil, err
-		}
-
-		return collections, nil
-	}
+type FetchCollectionResponse = FetchResponse[[]*Collection]
 
+func (platform *Platform) FetchCollections(
+	fetcher Work[Pagination, FetchCollectionResponse],
+	start_pagination Pagination,
+) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -44,15 +31,62 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
 		timeout:           time.Second * 2,
 	}
 
-	tasks := make(chan int)
+	tasks := make(chan Pagination)
 
-	// TODO: get number of page dynamically and change Fetcher signature
-	pages := 4
+	results, errors, done := asyncTaskRunner(
+		ctx,
+		tasks,
+		config,
+		fetcher,
+	)
 
-	results, errors, done := asyncTaskRunner(ctx, tasks, config, fetchWrapper)
+	var current_page uint64 = 0
 
-	for i := range pages {
-		tasks <- i * start_pagination.Limit
+	if start_pagination.Pages == 0 {
+		// do the first fetch
+		tasks <- start_pagination
+
+		select {
+		case response, ok := <-results:
+			if !ok {
+				break
+			}
+			platform.Collections = slices.Concat(platform.Collections, response.Response)
+
+			pages, err := calculatePages(&response.Pagination, start_pagination.Offset)
+			if err != nil {
+				return err
+			}
+			start_pagination.Pages = pages
+			start_pagination.Total = response.Pagination.Total
+			current_page++
+
+		case error, ok := <-errors:
+			if !ok {
+				return fmt.Errorf("Could not do first fetch to calculate pages: %v\n", error)
+			}
+
+		case <-ctx.Done():
+			break
+		case <-done:
+			break
+		}
+	}
+
+	page_offset, err := getPageByOffset(&start_pagination)
+	if err != nil {
+		return err
+	}
+
+	current_page += page_offset
+
+	fmt.Printf("Total pages: %v, Current page: %v\n", start_pagination.Pages, current_page)
+
+	for current_page <= start_pagination.Pages {
+		page := start_pagination
+		page.Offset = current_page * page.Limit
+		tasks <- page
+		current_page++
 	}
 
 	close(tasks)
@@ -60,11 +94,11 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
 loop:
 	for {
 		select {
-		case collection, ok := <-results:
+		case response, ok := <-results:
 			if !ok {
 				continue
 			}
-			platform.Collections = slices.Concat(platform.Collections, collection)
+			platform.Collections = slices.Concat(platform.Collections, response.Response)
 		case error, ok := <-errors:
 			if !ok {
 				continue
@@ -80,7 +114,7 @@ loop:
 
 	fmt.Printf("Collections: %v\n", len(platform.Collections))
 
-	err := BulkCreateNode(platform._conn, platform.Collections)
+	err = BulkCreateNode(platform._conn, platform.Collections)
 	if err != nil {
 		return err
 	}