perf: do a bulk insert in fetchCollection

Improve performance over inserting each item individually
This commit is contained in:
Alexander Navarro 2024-11-22 20:20:22 -03:00
parent 8c660053e5
commit 3cf643c83d
5 changed files with 118 additions and 12 deletions

View file

@ -1,5 +1,7 @@
package synchronizator
import "slices"
type PlatformClass interface {
// How to transform the struct into a node. It needs to return the class,
// name and a []byte representation of the metadata.
@ -24,7 +26,7 @@ type Platform struct {
collections []*Collection // Child nodes
}
type Fetcher = func(conn *DB, pagination Pagination) ([]*Collection, Pagination, error)
type Fetcher = func(pagination Pagination) ([]*Collection, Pagination, error)
type Pagination struct {
Total int
@ -41,14 +43,17 @@ var StartPagination = Pagination{
}
func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error {
collections, pagination, err := fetcher(platform._conn, start_pagination)
collections, pagination, err := fetcher(start_pagination)
if err != nil {
return err
}
platform.collections = collections
platform.collections = slices.Concat(platform.collections, collections)
if pagination.HasMore {
return platform.FetchCollections(fetcher, pagination)
}
err = BulkCreateNode(platform._conn, platform.collections)
return nil
}