synchronizator-go/examples/usage/main.go

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"time"

	_ "modernc.org/sqlite"

	synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
)

type PokeApiListResponse[T any] struct {
	Count    uint64 `json:"count"`
	Next     string `json:"next"`
	Previous string `json:"previous"`
	Results  []T    `json:"results"`
}

type Pokedex struct {
	Name string `json:"name"`
	Url  string `json:"url"`
}

type PokeApiPokedexResponse struct {
	Pokemons []Pokemon `json:"pokemon_entries"`
}

type PokemonSpecies struct {
	Name string `json:"name"`
	Url  string `json:"url"`
}

type Pokemon struct {
	Id             int            `json:"entry_number"`
	PokemonSpecies PokemonSpecies `json:"pokemon_species"`
}

func getPokedexs(
	ctx context.Context,
	pagination synchronizator.Pagination,
) (synchronizator.FetchCollectionResponse, error) {
	payload := synchronizator.FetchCollectionResponse{
		Pagination: pagination,
	}

	var collections []*synchronizator.Collection

	params := url.Values{}
	params.Add("offset", strconv.FormatUint(pagination.Offset, 10))
	params.Add("limit", strconv.FormatUint(pagination.Limit, 10))

	resp, err := http.Get("https://pokeapi.co/api/v2/pokedex?" + params.Encode())
	if err != nil {
		return payload, err
	}

	body, err := io.ReadAll(resp.Body)
	resp.Body.Close()

	var data PokeApiListResponse[Pokedex]
	err = json.Unmarshal(body, &data)
	if err != nil {
		return payload, err
	}

	collections = make([]*synchronizator.Collection, 0, len(data.Results))

	for _, pokedex := range data.Results {
		collection_name := "Pokedex_" + pokedex.Name
		metadata, err := json.Marshal(pokedex)
		collection := synchronizator.NewCollection(collection_name, metadata)
		if err != nil {
			return payload, err
		}
		collections = append(collections, collection)
	}

	// fmt.Println(data)

	payload.Offset += pagination.Limit
	payload.Total = data.Count

	payload.Response = collections

	return payload, nil
}

func getPokemons(
	ctx context.Context,
	pagination synchronizator.Pagination,
) (synchronizator.FetchNodesResponse, error) {
	payload := synchronizator.FetchNodesResponse{
		Pagination: pagination,
	}

	var pokemons []*synchronizator.Node

	pokedex, ok := ctx.Value("pokedex").(*Pokedex)

	if !ok {
		return payload, fmt.Errorf("Couldn't retreive pokedex from context!")
	}

	resp, err := http.Get(pokedex.Url)
	if err != nil {
		return payload, err
	}

	body, err := io.ReadAll(resp.Body)
	resp.Body.Close()

	var data PokeApiPokedexResponse
	err = json.Unmarshal(body, &data)
	if err != nil {
		return payload, err
	}

	pokemons = make([]*synchronizator.Node, 0, len(data.Pokemons))

	for _, pokemon := range data.Pokemons {
		metadata, err := json.Marshal(pokemon)
		node := synchronizator.NewNode(pokemon.PokemonSpecies.Name, metadata)
		if err != nil {
			return payload, err
		}
		pokemons = append(pokemons, node)
	}

	payload.Response = pokemons

	return payload, nil
}

func main() {
	start := time.Now()

	defer func() {
		elapsed := time.Now().Sub(start)
		fmt.Printf("\n\nExecution time took: %s", elapsed)
	}()

	connection, err := sql.Open("sqlite", "db.sql")
	if err != nil {
		fmt.Println(err)

		return
	}

	defer connection.Close()

	opts := synchronizator.DefaultOptions
	// opts.Log_level = synchronizator.DEBUG
	opts.DANGEROUSLY_DROP_TABLES = true

	sync, err := synchronizator.New(connection, opts)
	if err != nil {
		fmt.Println(err)

		return
	}

	pokeApi, err := sync.NewPlatform("pokeapi", nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	pagination := synchronizator.StartPagination
	pool_config := &synchronizator.WorkConfig{
		AmountOfWorkers: 5,
		MaxRetries:      1,
		BaseRetryTime:   time.Second * 2,
		RateLimit:       synchronizator.NewRateLimiter(5, time.Minute),
		Timeout:         time.Second * 2,
	}

	err = pokeApi.FetchCollections(ctx, getPokedexs, pagination, pool_config)
	if err != nil {
		fmt.Println(err)
		return
	}

	for _, collection := range pokeApi.Collections {
		if collection.IsDefault() {
			continue
		}

		pagination := synchronizator.StartPagination
		// It returns all the pokemons of 1 pokedex in a single request
		pagination.Pages = 1

		pokedex := &Pokedex{}
		err := json.Unmarshal(collection.GetMetadata(), pokedex)
		if err != nil {
			fmt.Println(err)
			return
		}

		ctx := context.WithValue(ctx, "pokedex", pokedex)

		err = collection.FetchNodes(ctx, getPokemons, pagination, pool_config)
		if err != nil {
			fmt.Println(err)
			return
		}
	}
}