synchronizator-go/pkg/platform.go

package synchronizator

import (
	"context"
	"fmt"
	"slices"
	"time"
)

// Utility struct to represent a collection of nodes, it's a [Node] itself so all
// the node's functionality is available.
type Platform struct {
	Node                      // Underlaying node info
	Collections []*Collection // Child nodes
}

type FetchCollectionResponse = FetchResponse[[]*Collection]

func (platform *Platform) FetchCollections(
	fetcher Work[Pagination, FetchCollectionResponse],
	start_pagination Pagination,
) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	config := &WorkConfig{
		amount_of_workers: 5,
		max_retries:       2,
		base_retry_time:   time.Second,
		rate_limit:        NewRateLimiter(5, time.Minute),
		timeout:           time.Second * 2,
	}

	tasks := make(chan Pagination)

	results, errors, done := asyncTaskRunner(
		ctx,
		tasks,
		config,
		fetcher,
	)

	var current_page uint64 = 0

	if start_pagination.Pages == 0 {
		// do the first fetch
		tasks <- start_pagination

		select {
		case response, ok := <-results:
			if !ok {
				break
			}
			platform.Collections = slices.Concat(platform.Collections, response.Response)

			pages, err := calculatePages(&response.Pagination, start_pagination.Offset)
			if err != nil {
				return err
			}
			start_pagination.Pages = pages
			start_pagination.Total = response.Pagination.Total
			current_page++

		case error, ok := <-errors:
			if !ok {
				return fmt.Errorf("Could not do first fetch to calculate pages: %v\n", error)
			}

		case <-ctx.Done():
			break
		case <-done:
			break
		}
	}

	page_offset, err := getPageByOffset(&start_pagination)
	if err != nil {
		return err
	}

	current_page += page_offset

	fmt.Printf("Total pages: %v, Current page: %v\n", start_pagination.Pages, current_page)

	for current_page <= start_pagination.Pages {
		page := start_pagination
		page.Offset = current_page * page.Limit
		tasks <- page
		current_page++
	}

	close(tasks)

loop:
	for {
		select {
		case response, ok := <-results:
			if !ok {
				continue
			}
			platform.Collections = slices.Concat(platform.Collections, response.Response)
		case error, ok := <-errors:
			if !ok {
				continue
			}

			fmt.Printf("There was an error: %v\n", error)
		case <-ctx.Done():
			break loop
		case <-done:
			break loop
		}
	}

	fmt.Printf("Collections: %v\n", len(platform.Collections))

	err = BulkCreateNode(platform._conn, platform.Collections)
	if err != nil {
		return err
	}

	for _, item := range platform.Collections {
		err := platform.AddRelationship(
			&Relationship{
				_class: "PLATFORM_HAS_COLLECTION",
				From:   platform.Id,
				To:     item.Id,
			})
		if err != nil {
			return err
		}
	}

	err = BulkCreateRelationships(platform._conn, platform._relationships)
	if err != nil {
		return err
	}

	return nil
}