package main import ( "database/sql" "encoding/json" "fmt" "io" "net/http" "net/url" "strconv" "time" _ "modernc.org/sqlite" synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" ) type PokeApiListResponse[T any] struct { Count int `json:"count"` Next string `json:"next"` Previous string `json:"previous"` Results []T `json:"results"` } type Pokedex struct { Name string `json:"name"` Url string `json:"url"` } type PokeApiPokedexResponse struct { Pokemons []Pokemon `json:"pokemon_entries"` } type PokemonSpecies struct { Name string `json:"name"` Url string `json:"url"` } type Pokemon struct { Id int `json:"entry_number"` PokemonSpecies PokemonSpecies `json:"pokemon_species"` } func getPokedexs( pagination synchronizator.Pagination, ) ([]*synchronizator.Collection, synchronizator.Pagination, error) { var collections []*synchronizator.Collection params := url.Values{} params.Add("offset", strconv.Itoa(pagination.Offset)) params.Add("limit", strconv.Itoa(pagination.Limit)) resp, err := http.Get("https://pokeapi.co/api/v2/pokedex?" + params.Encode()) if err != nil { return nil, pagination, err } body, err := io.ReadAll(resp.Body) resp.Body.Close() var data PokeApiListResponse[Pokedex] err = json.Unmarshal(body, &data) if err != nil { return nil, pagination, err } collections = make([]*synchronizator.Collection, 0, len(data.Results)) for _, pokedex := range data.Results { collection_name := "Pokedex_" + pokedex.Name metadata, err := json.Marshal(pokedex) collection := synchronizator.NewCollection(collection_name, metadata) if err != nil { return nil, pagination, err } collections = append(collections, collection) } // fmt.Println(data) pagination.Offset += pagination.Limit pagination.HasMore = data.Next != "" pagination.Total = data.Count return collections, pagination, nil } func getPokemons( metadata []byte, pagination synchronizator.Pagination, ) ([]*synchronizator.Node, synchronizator.Pagination, error) { var nodes []*synchronizator.Node pokedex := &Pokedex{} json.Unmarshal(metadata, pokedex) resp, err := http.Get(pokedex.Url) if err != nil { return nil, pagination, err } body, err := io.ReadAll(resp.Body) resp.Body.Close() var data PokeApiPokedexResponse err = json.Unmarshal(body, &data) if err != nil { return nil, pagination, err } nodes = make([]*synchronizator.Node, 0, len(data.Pokemons)) for _, pokemon := range data.Pokemons { metadata, err := json.Marshal(pokemon) node := synchronizator.NewNode(pokemon.PokemonSpecies.Name, metadata) if err != nil { return nil, pagination, err } nodes = append(nodes, node) } // fmt.Println(data) return nodes, pagination, nil } func main() { start := time.Now() defer func() { elapsed := time.Now().Sub(start) fmt.Printf("\n\nExecution time took: %s", elapsed) }() connection, err := sql.Open("sqlite", "db.sql") if err != nil { fmt.Println(err) return } defer connection.Close() opts := synchronizator.DefaultOptions // opts.Log_level = synchronizator.DEBUG opts.DANGEROUSLY_DROP_TABLES = true sync, err := synchronizator.New(connection, opts) if err != nil { fmt.Println(err) return } pokeApi, err := sync.NewPlatform("pokeapi", nil) if err != nil { fmt.Println(err) return } err = pokeApi.FetchCollections(getPokedexs, synchronizator.StartPagination) if err != nil { fmt.Println(err) return } for _, pokedex := range pokeApi.Collections { if pokedex.IsDefault() { continue } err = pokedex.FetchNodes(getPokemons, synchronizator.StartPagination) if err != nil { fmt.Println(err) return } } }