diff --git a/examples/mock_data/PokeApi/Get All Pokemons.bru b/examples/mock_data/PokeApi/Get All Pokemons.bru new file mode 100644 index 0000000..d6ac39a --- /dev/null +++ b/examples/mock_data/PokeApi/Get All Pokemons.bru @@ -0,0 +1,11 @@ +meta { + name: Get All Pokemons + type: http + seq: 2 +} + +get { + url: https://pokeapi.co/api/v2/pokemon/ + body: none + auth: none +} diff --git a/examples/mock_data/PokeApi/bruno.json b/examples/mock_data/PokeApi/bruno.json new file mode 100644 index 0000000..fa3cf16 --- /dev/null +++ b/examples/mock_data/PokeApi/bruno.json @@ -0,0 +1,9 @@ +{ + "version": "1", + "name": "PokeApi", + "type": "collection", + "ignore": [ + "node_modules", + ".git" + ] +} \ No newline at end of file diff --git a/examples/usage.go b/examples/usage.go index 3a9290a..0bc6ab5 100644 --- a/examples/usage.go +++ b/examples/usage.go @@ -2,24 +2,80 @@ package main import ( "database/sql" + "encoding/json" "fmt" + "io" + "net/http" + "net/url" + "strconv" _ "modernc.org/sqlite" synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" ) -type PokeApi struct{} - -func (pokeApi *PokeApi) ToNode() (string, []byte, error) { - return "POKEAPI", nil, nil +type PokeApiResponse[T any] struct { + Count int `json:"count"` + Next string `json:"next"` + Previous string `json:"previous"` + Results []T `json:"results"` } -func (pokeApi *PokeApi) FromNode(_class string, name string, metadata []byte) error { - if _class != "POKEAPI" { - return fmt.Errorf("invalid class %s", _class) +type Pokedex struct { + Name string `json:"name"` + Url string `json:"url"` +} + +type Pokemon struct { + Id int `json:"id"` + Name string `json:"name"` +} + +func getPokemons( + sync *synchronizator.DB, + pagination synchronizator.Pagination, +) ([]*synchronizator.Collection, synchronizator.Pagination, error) { + var collections []*synchronizator.Collection + + params := url.Values{} + params.Add("offset", strconv.Itoa(pagination.Offset)) + params.Add("limit", strconv.Itoa(pagination.Limit)) + + resp, err := http.Get("https://pokeapi.co/api/v2/pokedex?" + params.Encode()) + if err != nil { + return nil, pagination, err } - return nil + + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + + var data PokeApiResponse[Pokedex] + err = json.Unmarshal(body, &data) + if err != nil { + return nil, pagination, err + } + + collections = make([]*synchronizator.Collection, 0, len(data.Results)) + + // TODO: this writes to the database in each collection creation + // add better strategies, like returning a collection to the platform and the + // platform do a final bulk update + for _, pokedex := range data.Results { + collection_name := "Pokedex_" + pokedex.Name + collection, err := sync.NewCollection(collection_name, nil) + if err != nil { + return nil, pagination, err + } + collections = append(collections, collection) + } + + // fmt.Println(data) + + pagination.Offset += pagination.Limit + pagination.HasMore = data.Next != "" + pagination.Total = data.Count + + return collections, pagination, nil } func main() { @@ -43,6 +99,17 @@ func main() { return } - pokeApi := &PokeApi{} - sync.NewPlatform(pokeApi) + pokeApi, err := sync.NewPlatform("pokeapi", nil) + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(pokeApi) + + err = pokeApi.FetchCollections(getPokemons, synchronizator.StartPagination) + if err != nil { + fmt.Println(err) + return + } } diff --git a/pkg/collection.go b/pkg/collection.go index c796f38..20282de 100644 --- a/pkg/collection.go +++ b/pkg/collection.go @@ -54,16 +54,3 @@ func (collection *Collection) AddChild(node *Node) error { return nil } - -// Allows to retreive the saved information back into the user struct. This -// method will call the [NodeClass.FromNode] of the provided struct. -// -// Example: -// -// data := &Library{} -// if err := node.Unmarshall(data); err != nil { -// println(err) -// } -func (collection *Collection) Unmarshall(dst StandardNode) error { - return dst.FromNode("COLLECTION", collection.name, collection.metadata) -} diff --git a/pkg/node.go b/pkg/node.go index e002830..fd14907 100644 --- a/pkg/node.go +++ b/pkg/node.go @@ -1,29 +1,6 @@ package synchronizator -// A user representation of a node, this interface should be implemented by the user -// to provide the ability to parse the database node into a user defined -// struct that fulfills it's requirements. -// -// This interface is compatible with the [NodeClass] interface but it doesn't -// handle the class parameter as it's a static value provided by the -// StandardNode -type StandardNode interface { - // How to transform the struct into a node. It needs to return the class, - // name and a []byte representation of the metadata. - // - // - name: A user friendly name - // - metadata: Arbitrary data. This will be stored as a jsonb in the database - // - ToNode() (string, []byte, error) - - // How to transform a node into the struct. This method should modify the - // struct directly as it receives a pointer. - // - // - class: The class of the node, should not be modified to avoid inconsistencies. - // - name: A user friendly name - // - metadata: Arbitrary data. This is stored as a jsonb in the database - FromNode(string, string, []byte) error -} +type StandardNode interface{} // A node in the database. // It adds some helper methods to easily manage the node. @@ -112,19 +89,6 @@ func (node *Node) Delete() error { return node._conn.DeleteNode(node.Id) } -// Allows to retreive the saved information back into the user struct. This -// method will call the [NodeClass.FromNode] of the provided struct. -// -// Example: -// -// data := &Library{} -// if err := node.Unmarshall(data); err != nil { -// println(err) -// } -func (node *Node) Unmarshall(dst StandardNode) error { - return dst.FromNode(node._class, node.name, node.metadata) -} - // Returns the class of the node func (relationship *Relationship) GetClass() string { return relationship._class diff --git a/pkg/platform.go b/pkg/platform.go index 91003bd..ccb8135 100644 --- a/pkg/platform.go +++ b/pkg/platform.go @@ -23,3 +23,32 @@ type Platform struct { Node // Underlaying node info collections []*Collection // Child nodes } + +type Fetcher = func(conn *DB, pagination Pagination) ([]*Collection, Pagination, error) + +type Pagination struct { + Total int + HasMore bool + Limit int + Offset int +} + +var StartPagination = Pagination{ + Total: 0, + HasMore: false, + Limit: 10, + Offset: 0, +} + +func (platform *Platform) FetchCollections(fetcher Fetcher, start_pagination Pagination) error { + collections, pagination, err := fetcher(platform._conn, start_pagination) + if err != nil { + return err + } + platform.collections = collections + + if pagination.HasMore { + return platform.FetchCollections(fetcher, pagination) + } + return nil +} diff --git a/pkg/synchronizator.go b/pkg/synchronizator.go index e0a6ec4..54995a4 100644 --- a/pkg/synchronizator.go +++ b/pkg/synchronizator.go @@ -15,6 +15,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "time" ) @@ -168,23 +169,24 @@ func (conn *DB) Query(sql string, args ...any) (*sql.Rows, error) { // A collection is only a Node wrapper with some extended functionality to // manage multiple nodes. For more information see [DB.NewNode] method and the // [Platform] struct. -func (conn *DB) NewPlatform(data StandardNode) (*Platform, error) { +func (conn *DB) NewPlatform(name string, metadata []byte) (*Platform, error) { var platform *Platform err := conn.withTx(func(tx *sql.Tx) error { - node, err := conn.newNodewithTx(tx, "PLATFORM", data) + node, err := conn.newNodewithTx(tx, name, "PLATFORM", metadata) if err != nil { return err } collection, err := conn.newCollectionwithTx( tx, - &default_collection{platform_name: node.name}, + strings.ToUpper(name)+"_DEFAULT", + nil, ) if err != nil { return err } - platform := &Platform{ + platform = &Platform{ Node: *node, collections: []*Collection{collection}, } @@ -206,12 +208,12 @@ func (conn *DB) NewPlatform(data StandardNode) (*Platform, error) { // [Collection] struct. // // The operation is ran in a database transaction. -func (conn *DB) NewCollection(data StandardNode) (*Collection, error) { +func (conn *DB) NewCollection(name string, metadata []byte) (*Collection, error) { var collection *Collection err := conn.withTx(func(tx *sql.Tx) error { var err error - collection, err = conn.newCollectionwithTx(tx, data) + collection, err = conn.newCollectionwithTx(tx, name, metadata) return err }) @@ -223,8 +225,8 @@ func (conn *DB) NewCollection(data StandardNode) (*Collection, error) { // A collection is only a Node wrapper with some extended functionality to // manage multiple nodes. For more information see [DB.NewNode] method and the // [Collection] struct. -func (conn *DB) newCollectionwithTx(tx *sql.Tx, data StandardNode) (*Collection, error) { - node, err := conn.newNodewithTx(tx, "COLLECTION", data) +func (conn *DB) newCollectionwithTx(tx *sql.Tx, name string, metadata []byte) (*Collection, error) { + node, err := conn.newNodewithTx(tx, name, "COLLECTION", metadata) if err != nil { return nil, err } @@ -240,12 +242,12 @@ func (conn *DB) newCollectionwithTx(tx *sql.Tx, data StandardNode) (*Collection, // Creates a new node. // // The operation is ran in a database transaction. -func (conn *DB) NewNode(class string, data StandardNode) (*Node, error) { +func (conn *DB) NewNode(name string, class string, metadata []byte) (*Node, error) { var node *Node err := conn.withTx(func(tx *sql.Tx) error { var err error - node, err = conn.newNodewithTx(tx, class, data) + node, err = conn.newNodewithTx(tx, name, class, metadata) return err }) @@ -253,12 +255,12 @@ func (conn *DB) NewNode(class string, data StandardNode) (*Node, error) { } // Creates a new node -func (conn *DB) newNodewithTx(tx *sql.Tx, class string, data StandardNode) (*Node, error) { - name, metadata, err := data.ToNode() - if err != nil { - return nil, err - } - +func (conn *DB) newNodewithTx( + tx *sql.Tx, + name string, + class string, + metadata []byte, +) (*Node, error) { node := Node{ _conn: conn, _class: class, @@ -271,7 +273,7 @@ func (conn *DB) newNodewithTx(tx *sql.Tx, class string, data StandardNode) (*Nod sql := "INSERT INTO nodes (_class, name, metadata) VALUES ($1, $2, $3) RETURNING id;" - err = tx.QueryRow(sql, node._class, node.name, metadata).Scan(&node.Id) + err := tx.QueryRow(sql, node._class, node.name, metadata).Scan(&node.Id) if err != nil { return nil, err } @@ -280,12 +282,7 @@ func (conn *DB) newNodewithTx(tx *sql.Tx, class string, data StandardNode) (*Nod } // Updates a node with the provided id and data -func (conn *DB) UpdateNode(id int64, data StandardNode) (*Node, error) { - name, metadata, err := data.ToNode() - if err != nil { - return nil, err - } - +func (conn *DB) UpdateNode(id int64, name string, metadata []byte) (*Node, error) { node := Node{ _conn: conn, name: name,