synchronizator-go/pkg/platform.go

package synchronizator

import (
	"fmt"
	"slices"
	"time"
)

// Utility struct to represent a collection of nodes, it's a [Node] itself so all
// the node's functionality is available.
type Platform struct {
	Node                      // Underlaying node info
	Collections []*Collection // Child nodes
}

func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error {
	fetchWrapper := func(offset int) ([]*Collection, error) {
		fmt.Printf("Requesting offset: %v\n", offset)

		pagination := start_pagination

		pagination.Offset = offset
		collections, _, err := fetcher(pagination)
		if err != nil {
			return nil, err
		}

		return collections, nil
	}

	// 5 request per minute
	rate_limit := NewRateLimit(5, time.Minute)

	manager := createWorkerPool[int, []*Collection](5, rate_limit, fetchWrapper)

	// TODO: get number of page dynamically and change Fetcher signature
	pages := 4

	for i := range pages {
		err := manager.AddWork(i * start_pagination.Limit)
		if err != nil {
			return err
		}
	}

	for collections := range manager.GetWorkUnit() {
		platform.Collections = slices.Concat(platform.Collections, collections)
	}

	err := BulkCreateNode(platform._conn, platform.Collections)
	if err != nil {
		return err
	}

	for _, item := range platform.Collections {
		err := platform.AddRelationship(
			&Relationship{
				_class: "PLATFORM_HAS_COLLECTION",
				From:   platform.Id,
				To:     item.Id,
			})
		if err != nil {
			return err
		}
	}

	err = BulkCreateRelationships(platform._conn, platform._relationships)
	if err != nil {
		return err
	}

	return nil
}