package synchronizator import ( "context" "fmt" "slices" "time" ) // Utility struct to represent a collection of nodes, it's a [Node] itself so all // the node's functionality is available. type Platform struct { Node // Underlaying node info Collections []*Collection // Child nodes } type FetchCollectionResponse = FetchResponse[[]*Collection] func (platform *Platform) FetchCollections( fetcher Work[Pagination, FetchCollectionResponse], start_pagination Pagination, ) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() config := &WorkConfig{ amount_of_workers: 5, max_retries: 2, base_retry_time: time.Second, rate_limit: NewRateLimiter(5, time.Minute), timeout: time.Second * 2, } tasks := make(chan Pagination) results, errors, done := asyncTaskRunner( ctx, tasks, config, fetcher, ) var current_page uint64 = 0 if start_pagination.Pages == 0 { // do the first fetch tasks <- start_pagination select { case response, ok := <-results: if !ok { break } platform.Collections = slices.Concat(platform.Collections, response.Response) pages, err := calculatePages(&response.Pagination, start_pagination.Offset) if err != nil { return err } start_pagination.Pages = pages start_pagination.Total = response.Pagination.Total current_page++ case error, ok := <-errors: if !ok { return fmt.Errorf("Could not do first fetch to calculate pages: %v\n", error) } case <-ctx.Done(): break case <-done: break } } page_offset, err := getPageByOffset(&start_pagination) if err != nil { return err } current_page += page_offset fmt.Printf("Total pages: %v, Current page: %v\n", start_pagination.Pages, current_page) for current_page <= start_pagination.Pages { page := start_pagination page.Offset = current_page * page.Limit tasks <- page current_page++ } close(tasks) loop: for { select { case response, ok := <-results: if !ok { continue } platform.Collections = slices.Concat(platform.Collections, response.Response) case error, ok := <-errors: if !ok { continue } fmt.Printf("There was an error: %v\n", error) case <-ctx.Done(): break loop case <-done: break loop } } fmt.Printf("Collections: %v\n", len(platform.Collections)) err = BulkCreateNode(platform._conn, platform.Collections) if err != nil { return err } for _, item := range platform.Collections { err := platform.AddRelationship( &Relationship{ _class: "PLATFORM_HAS_COLLECTION", From: platform.Id, To: item.Id, }) if err != nil { return err } } err = BulkCreateRelationships(platform._conn, platform._relationships) if err != nil { return err } return nil }