alecodes/synchronizator-go
Archived
2
0
Fork 0
generated from alecodes/base-template
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

feat: update getNodes from collection to use worker pool

Browse source
This commit is contained in:
Alexander Navarro 2024-12-01 12:26:28 -03:00
parent 26b678b348
commit f58db2ecaa
2 changed files with 86 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

68
examples/usage.go
View file

@ -93,17 +93,24 @@ func getPokedexs(
}
func getPokemons(
metadata []byte,
ctx context.Context,
pagination synchronizator.Pagination,
) ([]*synchronizator.Node, synchronizator.Pagination, error) {
var nodes []*synchronizator.Node
) (synchronizator.FetchNodesResponse, error) {
payload := synchronizator.FetchNodesResponse{
Pagination: pagination,
}
pokedex := &Pokedex{}
json.Unmarshal(metadata, pokedex)
var pokemons []*synchronizator.Node
pokedex, ok := ctx.Value("pokedex").(*Pokedex)
if !ok {
return payload, fmt.Errorf("Couldn't retreive pokedex from context!")
}
resp, err := http.Get(pokedex.Url)
if err != nil {
return nil, pagination, err
return payload, err
}
body, err := io.ReadAll(resp.Body)
@ -112,23 +119,23 @@ func getPokemons(
var data PokeApiPokedexResponse
err = json.Unmarshal(body, &data)
if err != nil {
return nil, pagination, err
return payload, err
}
nodes = make([]*synchronizator.Node, 0, len(data.Pokemons))
pokemons = make([]*synchronizator.Node, 0, len(data.Pokemons))
for _, pokemon := range data.Pokemons {
metadata, err := json.Marshal(pokemon)
node := synchronizator.NewNode(pokemon.PokemonSpecies.Name, metadata)
if err != nil {
return nil, pagination, err
return payload, err
}
nodes = append(nodes, node)
pokemons = append(pokemons, node)
}
// fmt.Println(data)
payload.Response = pokemons
return nodes, pagination, nil
return payload, nil
}
func main() {
@ -171,7 +178,7 @@ func main() {
pagination := synchronizator.StartPagination
pool_config := &synchronizator.WorkConfig{
AmountOfWorkers: 5,
MaxRetries: 3,
MaxRetries: 1,
BaseRetryTime: time.Second * 2,
RateLimit: synchronizator.NewRateLimiter(5, time.Minute),
Timeout: time.Second * 2,
@ -183,15 +190,28 @@ func main() {
return
}
// for _, pokedex := range pokeApi.Collections {
// if pokedex.IsDefault() {
// continue
// }
//
// err = pokedex.FetchNodes(getPokemons, synchronizator.StartPagination)
// if err != nil {
// fmt.Println(err)
// return
// }
// }
for _, collection := range pokeApi.Collections {
if collection.IsDefault() {
continue
}
pagination := synchronizator.StartPagination
// It returns all the pokemons of 1 pokedex in a single request
pagination.Pages = 1
pokedex := &Pokedex{}
err := json.Unmarshal(collection.GetMetadata(), pokedex)
if err != nil {
fmt.Println(err)
return
}
ctx := context.WithValue(ctx, "pokedex", pokedex)
err = collection.FetchNodes(ctx, getPokemons, pagination, pool_config)
if err != nil {
fmt.Println(err)
return
}
}
}