package synchronizator import ( "context" "fmt" "slices" "time" ) // Utility struct to represent a collection of nodes, it's a [Node] itself so all // the node's functionality is available. type Platform struct { Node // Underlaying node info Collections []*Collection // Child nodes } func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error { fetchWrapper := func(offset int) ([]*Collection, error) { // fmt.Printf("Requesting offset: %v\n", offset) if offset == 10 { time.Sleep(time.Second * 5) } pagination := start_pagination pagination.Offset = offset collections, _, err := fetcher(pagination) if err != nil { return nil, err } return collections, nil } ctx, cancel := context.WithCancel(context.Background()) defer cancel() config := &WorkConfig{ max_workers: 5, max_retries: 2, base_retry_time: time.Second, rate_limit: NewRateLimit(5, time.Minute), timeout: time.Second * 2, } tasks := make(chan int) // TODO: get number of page dynamically and change Fetcher signature pages := 4 results, errors, done := asyncTaskRunner[int, []*Collection](ctx, tasks, config, fetchWrapper) for i := range pages { tasks <- i * start_pagination.Limit } close(tasks) loop: for { select { case collection, ok := <-results: if !ok { continue } platform.Collections = slices.Concat(platform.Collections, collection) case error, ok := <-errors: if !ok { continue } fmt.Printf("There was an error: %v\n", error) case <-ctx.Done(): break loop case <-done: break loop } } fmt.Printf("Collections: %v\n", len(platform.Collections)) err := BulkCreateNode(platform._conn, platform.Collections) if err != nil { return err } for _, item := range platform.Collections { err := platform.AddRelationship( &Relationship{ _class: "PLATFORM_HAS_COLLECTION", From: platform.Id, To: item.Id, }) if err != nil { return err } } err = BulkCreateRelationships(platform._conn, platform._relationships) if err != nil { return err } return nil }