feat: allow to fetch nodes

examples/mock_data/PokeApi/All Pokedex.bru

@@ -0,0 +1,11 @@
+meta {
+  name: All Pokedex
+  type: http
+  seq: 2
+}
+
+get {
+  url: https://pokeapi.co/api/v2/pokedex/
+  body: none
+  auth: none
+}

examples/mock_data/PokeApi/Get All Pokemons.bru

@@ -1,11 +0,0 @@
-meta {
-  name: Get All Pokemons
-  type: http
-  seq: 2
-}
-
-get {
-  url: https://pokeapi.co/api/v2/pokemon/
-  body: none
-  auth: none
-}

examples/mock_data/PokeApi/Pokemon from Pokedex.bru

@@ -0,0 +1,15 @@
+meta {
+  name: Pokemon from Pokedex
+  type: http
+  seq: 3
+}
+
+get {
+  url: https://pokeapi.co/api/v2/pokedex/:id
+  body: none
+  auth: none
+}
+
+params:path {
+  id: 1
+}

examples/usage.go

@@ -14,7 +14,7 @@ import (
 	synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
 )
 
-type PokeApiResponse[T any] struct {
+type PokeApiListResponse[T any] struct {
 	Count    int    `json:"count"`
 	Next     string `json:"next"`
 	Previous string `json:"previous"`
@@ -26,12 +26,21 @@ type Pokedex struct {
 	Url  string `json:"url"`
 }
 
-type Pokemon struct {
+type PokeApiPokedexResponse struct {
-	Id   int    `json:"id"`
+	Pokemons []Pokemon `json:"pokemon_entries"`
-	Name string `json:"name"`
 }
 
-func getPokemons(
+type PokemonSpecies struct {
+	Name string `json:"name"`
+	Url  string `json:"url"`
+}
+
+type Pokemon struct {
+	Id             int            `json:"entry_number"`
+	PokemonSpecies PokemonSpecies `json:"pokemon_species"`
+}
+
+func getPokedexs(
 	pagination synchronizator.Pagination,
 ) ([]*synchronizator.Collection, synchronizator.Pagination, error) {
 	var collections []*synchronizator.Collection
@@ -48,7 +57,7 @@ func getPokemons(
 	body, err := io.ReadAll(resp.Body)
 	resp.Body.Close()
 
-	var data PokeApiResponse[Pokedex]
+	var data PokeApiListResponse[Pokedex]
 	err = json.Unmarshal(body, &data)
 	if err != nil {
 		return nil, pagination, err
@@ -58,7 +67,8 @@ func getPokemons(
 
 	for _, pokedex := range data.Results {
 		collection_name := "Pokedex_" + pokedex.Name
-		collection := synchronizator.NewCollection(collection_name, nil)
+		metadata, err := json.Marshal(pokedex)
+		collection := synchronizator.NewCollection(collection_name, metadata)
 		if err != nil {
 			return nil, pagination, err
 		}
@@ -74,6 +84,45 @@ func getPokemons(
 	return collections, pagination, nil
 }
 
+func getPokemons(
+	metadata []byte,
+	pagination synchronizator.Pagination,
+) ([]*synchronizator.Node, synchronizator.Pagination, error) {
+	var nodes []*synchronizator.Node
+
+	pokedex := &Pokedex{}
+	json.Unmarshal(metadata, pokedex)
+
+	resp, err := http.Get(pokedex.Url)
+	if err != nil {
+		return nil, pagination, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	resp.Body.Close()
+
+	var data PokeApiPokedexResponse
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, pagination, err
+	}
+
+	nodes = make([]*synchronizator.Node, 0, len(data.Pokemons))
+
+	for _, pokemon := range data.Pokemons {
+		metadata, err := json.Marshal(pokemon)
+		node := synchronizator.NewNode(pokemon.PokemonSpecies.Name, metadata)
+		if err != nil {
+			return nil, pagination, err
+		}
+		nodes = append(nodes, node)
+	}
+
+	// fmt.Println(data)
+
+	return nodes, pagination, nil
+}
+
 func main() {
 	connection, err := sql.Open("sqlite", "db.sql")
 	if err != nil {
@@ -85,7 +134,7 @@ func main() {
 	defer connection.Close()
 
 	opts := synchronizator.DefaultOptions
-	opts.Log_level = synchronizator.DEBUG
+	// opts.Log_level = synchronizator.DEBUG
 	opts.DANGEROUSLY_DROP_TABLES = true
 
 	sync, err := synchronizator.New(connection, opts)
@@ -101,11 +150,21 @@ func main() {
 		return
 	}
 
-	err = pokeApi.FetchCollections(getPokemons, synchronizator.StartPagination)
+	err = pokeApi.FetchCollections(getPokedexs, synchronizator.StartPagination)
 	if err != nil {
 		fmt.Println(err)
 		return
 	}
 
-	fmt.Println(pokeApi)
+	for _, pokedex := range pokeApi.Collections {
+		if pokedex.IsDefault() {
+			continue
+		}
+
+		err = pokedex.FetchNodes(getPokemons, synchronizator.StartPagination)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+	}
 }

pkg/collection.go

@@ -2,6 +2,7 @@ package synchronizator
 
 import (
 	"fmt"
+	"slices"
 	"strings"
 )
 
@@ -26,6 +27,7 @@ func (collection *default_collection) FromNode(_class string, name string, metad
 type Collection struct {
 	Node               // Underlaying node info
 	childs     []*Node // Child nodes
+	is_default bool
 }
 
 // NewCollectionObject creates a new Collection instance without persisting it to the database.
@@ -39,6 +41,7 @@ func NewCollection(name string, metadata []byte) *Collection {
 			Id:       -1, // Use -1 to indicate not persisted
 		},
 		childs:     make([]*Node, 0),
+		is_default: false,
 	}
 }
 
@@ -68,3 +71,25 @@ func (collection *Collection) AddChild(node *Node) error {
 
 	return nil
 }
+
+func (collection *Collection) IsDefault() bool {
+	return collection.is_default
+}
+
+type NodeFetcher = func(metadata []byte, pagination Pagination) ([]*Node, Pagination, error)
+
+func (collection *Collection) FetchNodes(fetcher NodeFetcher, start_pagination Pagination) error {
+	nodes, pagination, err := fetcher(collection.GetMetadata(), start_pagination)
+	if err != nil {
+		return err
+	}
+	collection.childs = slices.Concat(collection.childs, nodes)
+
+	if pagination.HasMore {
+		return collection.FetchNodes(fetcher, pagination)
+	}
+
+	err = BulkCreateNode(collection._conn, collection.childs)
+
+	return nil
+}

pkg/node.go

@@ -20,6 +20,15 @@ type Node struct {
 	metadata       []byte          // Arbitrary data. This is stored as a jsonb in the database
 }
 
+func NewNode(name string, metadata []byte) *Node {
+	return &Node{
+		name:     name,
+		_class:   "COLLECTION",
+		metadata: metadata,
+		Id:       -1, // Use -1 to indicate not persisted
+	}
+}
+
 func (node *Node) GetClass() string {
 	return node._class
 }

pkg/platform.go

@@ -23,7 +23,7 @@ type PlatformClass interface {
 // the node's functionality is available.
 type Platform struct {
 	Node                      // Underlaying node info
-	collections []*Collection // Child nodes
+	Collections []*Collection // Child nodes
 }
 
 type Fetcher = func(pagination Pagination) ([]*Collection, Pagination, error)
@@ -47,13 +47,13 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag
 	if err != nil {
 		return err
 	}
-	platform.collections = slices.Concat(platform.collections, collections)
+	platform.Collections = slices.Concat(platform.Collections, collections)
 
 	if pagination.HasMore {
 		return platform.FetchCollections(fetcher, pagination)
 	}
 
-	err = BulkCreateNode(platform._conn, platform.collections)
+	err = BulkCreateNode(platform._conn, platform.Collections)
 
 	return nil
 }

pkg/synchronizator.go

@@ -186,9 +186,11 @@ func (conn *DB) NewPlatform(name string, metadata []byte) (*Platform, error) {
 			return err
 		}
 
+		collection.is_default = true
+
 		platform = &Platform{
 			Node:        *node,
-			collections: []*Collection{collection},
+			Collections: []*Collection{collection},
 		}
 
 		_, err = conn.addRelationwithTx(tx, platform.Id, &collection_relation{}, collection.Id)
@@ -234,6 +236,7 @@ func (conn *DB) newCollectionwithTx(tx *sql.Tx, name string, metadata []byte) (*
 	collection := &Collection{
 		Node:       *node,
 		childs:     make([]*Node, 0),
+		is_default: false,
 	}
 
 	return collection, err