From 8af94161ab695b2e577136c50f2e50954e9ca3b3 Mon Sep 17 00:00:00 2001 From: aleidk Date: Fri, 22 Nov 2024 21:05:15 -0300 Subject: [PATCH] feat: allow to fetch nodes --- examples/mock_data/PokeApi/All Pokedex.bru | 11 +++ .../mock_data/PokeApi/Get All Pokemons.bru | 11 --- .../PokeApi/Pokemon from Pokedex.bru | 15 ++++ examples/usage.go | 79 ++++++++++++++++--- pkg/collection.go | 31 +++++++- pkg/node.go | 9 +++ pkg/platform.go | 6 +- pkg/synchronizator.go | 9 ++- 8 files changed, 141 insertions(+), 30 deletions(-) create mode 100644 examples/mock_data/PokeApi/All Pokedex.bru delete mode 100644 examples/mock_data/PokeApi/Get All Pokemons.bru create mode 100644 examples/mock_data/PokeApi/Pokemon from Pokedex.bru diff --git a/examples/mock_data/PokeApi/All Pokedex.bru b/examples/mock_data/PokeApi/All Pokedex.bru new file mode 100644 index 0000000..d790bc0 --- /dev/null +++ b/examples/mock_data/PokeApi/All Pokedex.bru @@ -0,0 +1,11 @@ +meta { + name: All Pokedex + type: http + seq: 2 +} + +get { + url: https://pokeapi.co/api/v2/pokedex/ + body: none + auth: none +} diff --git a/examples/mock_data/PokeApi/Get All Pokemons.bru b/examples/mock_data/PokeApi/Get All Pokemons.bru deleted file mode 100644 index d6ac39a..0000000 --- a/examples/mock_data/PokeApi/Get All Pokemons.bru +++ /dev/null @@ -1,11 +0,0 @@ -meta { - name: Get All Pokemons - type: http - seq: 2 -} - -get { - url: https://pokeapi.co/api/v2/pokemon/ - body: none - auth: none -} diff --git a/examples/mock_data/PokeApi/Pokemon from Pokedex.bru b/examples/mock_data/PokeApi/Pokemon from Pokedex.bru new file mode 100644 index 0000000..213573d --- /dev/null +++ b/examples/mock_data/PokeApi/Pokemon from Pokedex.bru @@ -0,0 +1,15 @@ +meta { + name: Pokemon from Pokedex + type: http + seq: 3 +} + +get { + url: https://pokeapi.co/api/v2/pokedex/:id + body: none + auth: none +} + +params:path { + id: 1 +} diff --git a/examples/usage.go b/examples/usage.go index 7680766..dbf380b 100644 --- a/examples/usage.go +++ b/examples/usage.go @@ -14,7 +14,7 @@ import ( synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" ) -type PokeApiResponse[T any] struct { +type PokeApiListResponse[T any] struct { Count int `json:"count"` Next string `json:"next"` Previous string `json:"previous"` @@ -26,12 +26,21 @@ type Pokedex struct { Url string `json:"url"` } -type Pokemon struct { - Id int `json:"id"` - Name string `json:"name"` +type PokeApiPokedexResponse struct { + Pokemons []Pokemon `json:"pokemon_entries"` } -func getPokemons( +type PokemonSpecies struct { + Name string `json:"name"` + Url string `json:"url"` +} + +type Pokemon struct { + Id int `json:"entry_number"` + PokemonSpecies PokemonSpecies `json:"pokemon_species"` +} + +func getPokedexs( pagination synchronizator.Pagination, ) ([]*synchronizator.Collection, synchronizator.Pagination, error) { var collections []*synchronizator.Collection @@ -48,7 +57,7 @@ func getPokemons( body, err := io.ReadAll(resp.Body) resp.Body.Close() - var data PokeApiResponse[Pokedex] + var data PokeApiListResponse[Pokedex] err = json.Unmarshal(body, &data) if err != nil { return nil, pagination, err @@ -58,7 +67,8 @@ func getPokemons( for _, pokedex := range data.Results { collection_name := "Pokedex_" + pokedex.Name - collection := synchronizator.NewCollection(collection_name, nil) + metadata, err := json.Marshal(pokedex) + collection := synchronizator.NewCollection(collection_name, metadata) if err != nil { return nil, pagination, err } @@ -74,6 +84,45 @@ func getPokemons( return collections, pagination, nil } +func getPokemons( + metadata []byte, + pagination synchronizator.Pagination, +) ([]*synchronizator.Node, synchronizator.Pagination, error) { + var nodes []*synchronizator.Node + + pokedex := &Pokedex{} + json.Unmarshal(metadata, pokedex) + + resp, err := http.Get(pokedex.Url) + if err != nil { + return nil, pagination, err + } + + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + + var data PokeApiPokedexResponse + err = json.Unmarshal(body, &data) + if err != nil { + return nil, pagination, err + } + + nodes = make([]*synchronizator.Node, 0, len(data.Pokemons)) + + for _, pokemon := range data.Pokemons { + metadata, err := json.Marshal(pokemon) + node := synchronizator.NewNode(pokemon.PokemonSpecies.Name, metadata) + if err != nil { + return nil, pagination, err + } + nodes = append(nodes, node) + } + + // fmt.Println(data) + + return nodes, pagination, nil +} + func main() { connection, err := sql.Open("sqlite", "db.sql") if err != nil { @@ -85,7 +134,7 @@ func main() { defer connection.Close() opts := synchronizator.DefaultOptions - opts.Log_level = synchronizator.DEBUG + // opts.Log_level = synchronizator.DEBUG opts.DANGEROUSLY_DROP_TABLES = true sync, err := synchronizator.New(connection, opts) @@ -101,11 +150,21 @@ func main() { return } - err = pokeApi.FetchCollections(getPokemons, synchronizator.StartPagination) + err = pokeApi.FetchCollections(getPokedexs, synchronizator.StartPagination) if err != nil { fmt.Println(err) return } - fmt.Println(pokeApi) + for _, pokedex := range pokeApi.Collections { + if pokedex.IsDefault() { + continue + } + + err = pokedex.FetchNodes(getPokemons, synchronizator.StartPagination) + if err != nil { + fmt.Println(err) + return + } + } } diff --git a/pkg/collection.go b/pkg/collection.go index 79c554f..e4dfa05 100644 --- a/pkg/collection.go +++ b/pkg/collection.go @@ -2,6 +2,7 @@ package synchronizator import ( "fmt" + "slices" "strings" ) @@ -24,8 +25,9 @@ func (collection *default_collection) FromNode(_class string, name string, metad // Utility struct to represent a collection of nodes, it's a [Node] itself so all // the node's functionality is available. type Collection struct { - Node // Underlaying node info - childs []*Node // Child nodes + Node // Underlaying node info + childs []*Node // Child nodes + is_default bool } // NewCollectionObject creates a new Collection instance without persisting it to the database. @@ -38,7 +40,8 @@ func NewCollection(name string, metadata []byte) *Collection { metadata: metadata, Id: -1, // Use -1 to indicate not persisted }, - childs: make([]*Node, 0), + childs: make([]*Node, 0), + is_default: false, } } @@ -68,3 +71,25 @@ func (collection *Collection) AddChild(node *Node) error { return nil } + +func (collection *Collection) IsDefault() bool { + return collection.is_default +} + +type NodeFetcher = func(metadata []byte, pagination Pagination) ([]*Node, Pagination, error) + +func (collection *Collection) FetchNodes(fetcher NodeFetcher, start_pagination Pagination) error { + nodes, pagination, err := fetcher(collection.GetMetadata(), start_pagination) + if err != nil { + return err + } + collection.childs = slices.Concat(collection.childs, nodes) + + if pagination.HasMore { + return collection.FetchNodes(fetcher, pagination) + } + + err = BulkCreateNode(collection._conn, collection.childs) + + return nil +} diff --git a/pkg/node.go b/pkg/node.go index 4a6e736..9ce83a0 100644 --- a/pkg/node.go +++ b/pkg/node.go @@ -20,6 +20,15 @@ type Node struct { metadata []byte // Arbitrary data. This is stored as a jsonb in the database } +func NewNode(name string, metadata []byte) *Node { + return &Node{ + name: name, + _class: "COLLECTION", + metadata: metadata, + Id: -1, // Use -1 to indicate not persisted + } +} + func (node *Node) GetClass() string { return node._class } diff --git a/pkg/platform.go b/pkg/platform.go index 41ff26d..ca0f5df 100644 --- a/pkg/platform.go +++ b/pkg/platform.go @@ -23,7 +23,7 @@ type PlatformClass interface { // the node's functionality is available. type Platform struct { Node // Underlaying node info - collections []*Collection // Child nodes + Collections []*Collection // Child nodes } type Fetcher = func(pagination Pagination) ([]*Collection, Pagination, error) @@ -47,13 +47,13 @@ func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pag if err != nil { return err } - platform.collections = slices.Concat(platform.collections, collections) + platform.Collections = slices.Concat(platform.Collections, collections) if pagination.HasMore { return platform.FetchCollections(fetcher, pagination) } - err = BulkCreateNode(platform._conn, platform.collections) + err = BulkCreateNode(platform._conn, platform.Collections) return nil } diff --git a/pkg/synchronizator.go b/pkg/synchronizator.go index 370229a..f1bfb93 100644 --- a/pkg/synchronizator.go +++ b/pkg/synchronizator.go @@ -186,9 +186,11 @@ func (conn *DB) NewPlatform(name string, metadata []byte) (*Platform, error) { return err } + collection.is_default = true + platform = &Platform{ Node: *node, - collections: []*Collection{collection}, + Collections: []*Collection{collection}, } _, err = conn.addRelationwithTx(tx, platform.Id, &collection_relation{}, collection.Id) @@ -232,8 +234,9 @@ func (conn *DB) newCollectionwithTx(tx *sql.Tx, name string, metadata []byte) (* } collection := &Collection{ - Node: *node, - childs: make([]*Node, 0), + Node: *node, + childs: make([]*Node, 0), + is_default: false, } return collection, err