generated from alecodes/base-template
106 lines
2.1 KiB
Go
106 lines
2.1 KiB
Go
package synchronizator
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
)
|
|
|
|
// Utility struct to represent a collection of nodes, it's a [Node] itself so all
|
|
// the node's functionality is available.
|
|
type Platform struct {
|
|
Node // Underlaying node info
|
|
Collections []*Collection // Child nodes
|
|
}
|
|
|
|
func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error {
|
|
fetchWrapper := func(offset int) ([]*Collection, error) {
|
|
// fmt.Printf("Requesting offset: %v\n", offset)
|
|
|
|
if offset == 10 {
|
|
time.Sleep(time.Second * 5)
|
|
}
|
|
|
|
pagination := start_pagination
|
|
|
|
pagination.Offset = offset
|
|
collections, _, err := fetcher(pagination)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return collections, nil
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
config := &WorkConfig{
|
|
amount_of_workers: 5,
|
|
max_retries: 2,
|
|
base_retry_time: time.Second,
|
|
rate_limit: NewRateLimiter(5, time.Minute),
|
|
timeout: time.Second * 2,
|
|
}
|
|
|
|
tasks := make(chan int)
|
|
|
|
// TODO: get number of page dynamically and change Fetcher signature
|
|
pages := 4
|
|
|
|
results, errors, done := asyncTaskRunner(ctx, tasks, config, fetchWrapper)
|
|
|
|
for i := range pages {
|
|
tasks <- i * start_pagination.Limit
|
|
}
|
|
|
|
close(tasks)
|
|
|
|
loop:
|
|
for {
|
|
select {
|
|
case collection, ok := <-results:
|
|
if !ok {
|
|
continue
|
|
}
|
|
platform.Collections = slices.Concat(platform.Collections, collection)
|
|
case error, ok := <-errors:
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
fmt.Printf("There was an error: %v\n", error)
|
|
case <-ctx.Done():
|
|
break loop
|
|
case <-done:
|
|
break loop
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Collections: %v\n", len(platform.Collections))
|
|
|
|
err := BulkCreateNode(platform._conn, platform.Collections)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, item := range platform.Collections {
|
|
err := platform.AddRelationship(
|
|
&Relationship{
|
|
_class: "PLATFORM_HAS_COLLECTION",
|
|
From: platform.Id,
|
|
To: item.Id,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
err = BulkCreateRelationships(platform._conn, platform._relationships)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|