feat: implement platform handler creation

also refactor public and internal api to support transaction between
multiple methods

#2
This commit is contained in:
Alexander Navarro 2024-11-18 16:51:09 -03:00
parent b2d8dadcee
commit 01086d12c9
7 changed files with 423 additions and 253 deletions

222
examples/usage.bkp.go Normal file
View file

@ -0,0 +1,222 @@
package main
import (
"database/sql"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
_ "modernc.org/sqlite"
)
type ProgrammingLanguage struct {
Name string
}
func (language *ProgrammingLanguage) ToNode() (string, string, []byte, error) {
metadata, err := json.Marshal("{\"test\": \"foo\"}")
if err != nil {
return "", "", nil, err
}
return "PROGRAMMING_LANGUAGE", language.Name, metadata, nil
}
func (language *ProgrammingLanguage) FromNode(_class string, name string, metadata []byte) error {
if _class != "PROGRAMMING_LANGUAGE" {
return fmt.Errorf("invalid class %s", _class)
}
language.Name = name
return nil
}
type Library struct {
Name string `json:"name"`
Category string `json:"category"`
Metadata map[string]interface{} `json:"metadata"`
}
func (library *Library) ToNode() (string, string, []byte, error) {
metadata, err := json.Marshal(library.Metadata)
if err != nil {
return "", "", nil, err
}
return "LIBRARY", library.Name, metadata, nil
}
func (library *Library) FromNode(_class string, name string, metadata []byte) error {
if _class != "LIBRARY" {
return fmt.Errorf("invalid class %s", _class)
}
if err := json.Unmarshal(metadata, &library.Metadata); err != nil {
return err
}
library.Name = name
return nil
}
type (
BelognsTo struct{}
IsSame struct{}
)
func main2() {
connection, err := sql.Open("sqlite", "db.sql")
if err != nil {
fmt.Println(err)
return
}
defer connection.Close()
opts := synchronizator.DefaultOptions
// opts.Log_level = synchronizator.DEBUG
opts.DANGEROUSLY_DROP_TABLES = true
sync, err := synchronizator.New(connection, opts)
if err != nil {
fmt.Println(err)
return
}
languages, err := loadData()
if err != nil {
fmt.Println(err)
}
for language, libraries := range languages {
_, err := generateCollection(
&ProgrammingLanguage{Name: strings.ToUpper(language)},
libraries,
sync,
)
if err != nil {
println(err)
}
// fmt.Fprintf(
// os.Stderr,
// "libraries_collection%+v\n",
// libraries_collection,
// )
}
golang, err := sync.GetNode(1)
if err != nil {
println(err)
}
fmt.Println("%v", golang)
relationships, err := golang.GetOutRelations()
if err != nil {
panic(err)
}
for _, relationship := range relationships {
fmt.Printf("%v -> %v -> %v\n", relationship.From, relationship.GetClass(), relationship.To)
}
}
// generateCollection Main example of the usage of the synchronizator package
func generateCollection(
language *ProgrammingLanguage,
libraries []Library,
sync *synchronizator.DB,
) (*synchronizator.Collection, error) {
language_libraries, err := sync.NewCollection(language)
if err != nil {
return nil, err
}
for _, library := range libraries {
node, err := sync.NewNode(&library)
if err != nil {
return nil, err
}
data := &Library{}
if err := node.Unmarshall(data); err != nil {
println(err)
}
if err := language_libraries.AddChild(node); err != nil {
return nil, err
}
}
return language_libraries, nil
}
func loadData() (map[string][]Library, error) {
// Find all CSV files
files, err := filepath.Glob("examples/mock_data/*.csv")
if err != nil {
return nil, fmt.Errorf("failed to glob files: %w", err)
}
result := make(map[string][]Library)
for _, file := range files {
// Load CSV file
libraries, err := processCSVFile(file)
if err != nil {
return nil, fmt.Errorf("failed to process %s: %w", file, err)
}
// Use base filename without extension as language_name
language_name := filepath.Base(file)
language_name = language_name[:len(language_name)-len(filepath.Ext(language_name))]
result[language_name] = libraries
}
return result, nil
}
func processCSVFile(filename string) ([]Library, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
defer file.Close()
reader := csv.NewReader(file)
// Skip header
_, err = reader.Read()
if err != nil {
return nil, err
}
var libraries []Library
// Read records
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
// Parse metadata JSON
var metadata map[string]interface{}
if err := json.Unmarshal([]byte(record[2]), &metadata); err != nil {
return nil, fmt.Errorf("failed to parse metadata: %w", err)
}
library := Library{
Name: record[0],
Category: record[1],
Metadata: metadata,
}
libraries = append(libraries, library)
}
return libraries, nil
}

View file

@ -2,68 +2,26 @@ package main
import ( import (
"database/sql" "database/sql"
"encoding/csv"
"encoding/json"
"fmt" "fmt"
"io"
"os" _ "modernc.org/sqlite"
"path/filepath"
"strings"
synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
_ "modernc.org/sqlite"
) )
type ProgrammingLanguage struct { type PokeApi struct{}
Name string
func (pokeApi *PokeApi) ToNode() (string, []byte, error) {
return "POKEAPI", nil, nil
} }
func (language *ProgrammingLanguage) ToNode() (string, string, []byte, error) { func (pokeApi *PokeApi) FromNode(_class string, name string, metadata []byte) error {
metadata, err := json.Marshal("{\"test\": \"foo\"}") if _class != "POKEAPI" {
if err != nil {
return "", "", nil, err
}
return "PROGRAMMING_LANGUAGE", language.Name, metadata, nil
}
func (language *ProgrammingLanguage) FromNode(_class string, name string, metadata []byte) error {
if _class != "PROGRAMMING_LANGUAGE" {
return fmt.Errorf("invalid class %s", _class) return fmt.Errorf("invalid class %s", _class)
} }
language.Name = name
return nil return nil
} }
type Library struct {
Name string `json:"name"`
Category string `json:"category"`
Metadata map[string]interface{} `json:"metadata"`
}
func (library *Library) ToNode() (string, string, []byte, error) {
metadata, err := json.Marshal(library.Metadata)
if err != nil {
return "", "", nil, err
}
return "LIBRARY", library.Name, metadata, nil
}
func (library *Library) FromNode(_class string, name string, metadata []byte) error {
if _class != "LIBRARY" {
return fmt.Errorf("invalid class %s", _class)
}
if err := json.Unmarshal(metadata, &library.Metadata); err != nil {
return err
}
library.Name = name
return nil
}
type (
BelognsTo struct{}
IsSame struct{}
)
func main() { func main() {
connection, err := sql.Open("sqlite", "db.sql") connection, err := sql.Open("sqlite", "db.sql")
if err != nil { if err != nil {
@ -85,138 +43,6 @@ func main() {
return return
} }
languages, err := loadData() pokeApi := &PokeApi{}
if err != nil { sync.NewPlatform(pokeApi)
fmt.Println(err)
}
for language, libraries := range languages {
_, err := generateCollection(
&ProgrammingLanguage{Name: strings.ToUpper(language)},
libraries,
sync,
)
if err != nil {
println(err)
}
// fmt.Fprintf(
// os.Stderr,
// "libraries_collection%+v\n",
// libraries_collection,
// )
}
golang, err := sync.GetNode(1)
if err != nil {
println(err)
}
fmt.Println("%v", golang)
relationships, err := golang.GetOutRelations()
if err != nil {
panic(err)
}
for _, relationship := range relationships {
fmt.Printf("%v -> %v -> %v\n", relationship.From, relationship.GetClass(), relationship.To)
}
}
// generateCollection Main example of the usage of the synchronizator package
func generateCollection(
language *ProgrammingLanguage,
libraries []Library,
sync *synchronizator.DB,
) (*synchronizator.Collection, error) {
language_libraries, err := sync.NewCollection(language)
if err != nil {
return nil, err
}
for _, library := range libraries {
node, err := sync.NewNode(&library)
if err != nil {
return nil, err
}
data := &Library{}
if err := node.Unmarshall(data); err != nil {
println(err)
}
if err := language_libraries.AddChild(node); err != nil {
return nil, err
}
}
return language_libraries, nil
}
func loadData() (map[string][]Library, error) {
// Find all CSV files
files, err := filepath.Glob("examples/mock_data/*.csv")
if err != nil {
return nil, fmt.Errorf("failed to glob files: %w", err)
}
result := make(map[string][]Library)
for _, file := range files {
// Load CSV file
libraries, err := processCSVFile(file)
if err != nil {
return nil, fmt.Errorf("failed to process %s: %w", file, err)
}
// Use base filename without extension as language_name
language_name := filepath.Base(file)
language_name = language_name[:len(language_name)-len(filepath.Ext(language_name))]
result[language_name] = libraries
}
return result, nil
}
func processCSVFile(filename string) ([]Library, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
defer file.Close()
reader := csv.NewReader(file)
// Skip header
_, err = reader.Read()
if err != nil {
return nil, err
}
var libraries []Library
// Read records
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
// Parse metadata JSON
var metadata map[string]interface{}
if err := json.Unmarshal([]byte(record[2]), &metadata); err != nil {
return nil, fmt.Errorf("failed to parse metadata: %w", err)
}
library := Library{
Name: record[0],
Category: record[1],
Metadata: metadata,
}
libraries = append(libraries, library)
}
return libraries, nil
} }

View file

@ -1,6 +1,25 @@
package synchronizator package synchronizator
import "fmt" import (
"fmt"
"strings"
)
type default_collection struct {
platform_name string
}
func (collection *default_collection) ToNode() (string, []byte, error) {
platform_name := strings.ToUpper(collection.platform_name)
return platform_name + "_DEFAULT", nil, nil
}
func (collection *default_collection) FromNode(_class string, name string, metadata []byte) error {
if _class != "DEFAULT" {
return fmt.Errorf("invalid class %s", _class)
}
return nil
}
// Utility struct to represent a collection of nodes, it's a [Node] itself so all // Utility struct to represent a collection of nodes, it's a [Node] itself so all
// the node's functionality is available. // the node's functionality is available.
@ -35,3 +54,16 @@ func (collection *Collection) AddChild(node *Node) error {
return nil return nil
} }
// Allows to retreive the saved information back into the user struct. This
// method will call the [NodeClass.FromNode] of the provided struct.
//
// Example:
//
// data := &Library{}
// if err := node.Unmarshall(data); err != nil {
// println(err)
// }
func (collection *Collection) Unmarshall(dst StandardNode) error {
return dst.FromNode("COLLECTION", collection.name, collection.metadata)
}

View file

@ -4,46 +4,22 @@ package synchronizator
// to provide the ability to parse the database node into a user defined // to provide the ability to parse the database node into a user defined
// struct that fulfills it's requirements. // struct that fulfills it's requirements.
// //
// Example usage: // This interface is compatible with the [NodeClass] interface but it doesn't
// // handle the class parameter as it's a static value provided by the
// type Library struct { // StandardNode
// Name string `json:"name"` type StandardNode interface {
// Category string `json:"category"`
// Metadata map[string]interface{} `json:"metadata"`
// }
//
// func (library *Library) ToNode() (string, string, []byte, error) {
// metadata, err := json.Marshal(library.Metadata)
// if err != nil {
// return "", "", nil, err
// }
// return "LIBRARY", library.Name, metadata, nil
// }
//
// func (library *Library) FromNode(_class string, name string, metadata []byte) error {
// if _class != "LIBRARY" {
// return fmt.Errorf("invalid class %s", _class)
// }
// if err := json.Unmarshal(metadata, &library.Metadata); err != nil {
// return err
// }
// library.Name = name
// return nil
// }
type NodeClass interface {
// How to transform the struct into a node. It needs to return the class, // How to transform the struct into a node. It needs to return the class,
// name and a []byte representation of the metadata. // name and a []byte representation of the metadata.
// //
// - class: Is used for classification and query pourposes. It's recomended to provide a constante string to increase consistency.
// - name: A user friendly name // - name: A user friendly name
// - metadata: Arbitrary data. This will be stored as a jsonb in the database // - metadata: Arbitrary data. This will be stored as a jsonb in the database
// //
ToNode() (string, string, []byte, error) ToNode() (string, []byte, error)
// How to transform a node into the struct. This method should modify the // How to transform a node into the struct. This method should modify the
// struct directly as it receives a pointer. // struct directly as it receives a pointer.
// //
// - class: Is used for classification and query pourposes. // - class: The class of the node, should not be modified to avoid inconsistencies.
// - name: A user friendly name // - name: A user friendly name
// - metadata: Arbitrary data. This is stored as a jsonb in the database // - metadata: Arbitrary data. This is stored as a jsonb in the database
FromNode(string, string, []byte) error FromNode(string, string, []byte) error
@ -60,11 +36,11 @@ type Node struct {
metadata []byte // Arbitrary data. This is stored as a jsonb in the database metadata []byte // Arbitrary data. This is stored as a jsonb in the database
} }
// Creates a new relation of type RelationshipClass to the node with the // Creates a new relation of type StandardRelationship to the node with the
// provided id. An error is returned if the relation already exists. // provided id. An error is returned if the relation already exists.
// //
// This method is a wrapper around the AddRelation method of the connection. // This method is a wrapper around the AddRelation method of the connection.
func (node *Node) AddRelation(relation RelationshipClass, to int64) (*Relationship, error) { func (node *Node) AddRelation(relation StandardRelationship, to int64) (*Relationship, error) {
return node._conn.AddRelation(node.Id, relation, to) return node._conn.AddRelation(node.Id, relation, to)
} }
@ -145,6 +121,11 @@ func (node *Node) Delete() error {
// if err := node.Unmarshall(data); err != nil { // if err := node.Unmarshall(data); err != nil {
// println(err) // println(err)
// } // }
func (node *Node) Unmarshall(dst NodeClass) error { func (node *Node) Unmarshall(dst StandardNode) error {
return dst.FromNode(node._class, node.name, node.metadata) return dst.FromNode(node._class, node.name, node.metadata)
} }
// Returns the class of the node
func (relationship *Relationship) GetClass() string {
return relationship._class
}

25
pkg/platform.go Normal file
View file

@ -0,0 +1,25 @@
package synchronizator
type PlatformClass interface {
// How to transform the struct into a node. It needs to return the class,
// name and a []byte representation of the metadata.
//
// - name: A user friendly name
// - metadata: Arbitrary data. This will be stored as a jsonb in the database
//
ToNode() (string, []byte, error)
// How to transform a node into the struct. This method should modify the
// struct directly as it receives a pointer.
//
// - name: A user friendly name
// - metadata: Arbitrary data. This is stored as a jsonb in the database
FromNode(string, []byte) error
}
// Utility struct to represent a collection of nodes, it's a [Node] itself so all
// the node's functionality is available.
type Platform struct {
Node // Underlaying node info
collections []*Collection // Child nodes
}

View file

@ -18,7 +18,7 @@ package synchronizator
// } // }
// return nil // return nil
// } // }
type RelationshipClass interface { type StandardRelationship interface {
// How to transform the struct into a collection. It needs to return the class, // How to transform the struct into a collection. It needs to return the class,
// and a []byte representation of the metadata. // and a []byte representation of the metadata.
// //
@ -44,7 +44,3 @@ type Relationship struct {
To int64 // To what node this relation goes to To int64 // To what node this relation goes to
Metadata []byte // Arbitrary data. This is stored as a jsonb in the database Metadata []byte // Arbitrary data. This is stored as a jsonb in the database
} }
func (relationship *Relationship) GetClass() string {
return relationship._class
}

View file

@ -4,6 +4,10 @@
// It does so implementing a graph database representing the relation of the // It does so implementing a graph database representing the relation of the
// different entities of data called "nodes", helping you find, create or // different entities of data called "nodes", helping you find, create or
// delete the "equivalent" entities in the different sources. // delete the "equivalent" entities in the different sources.
//
// In this library we use the following nomemclature:
// - [struct_name]: The representation of the element in the database
// - [struct_name]Class: An interface so the users can create a custom representation of the element that make's sence in their application. The interface needs to provide a way to transform the struct into a node and viceversa.
package synchronizator package synchronizator
import ( import (
@ -133,6 +137,21 @@ func (conn *DB) bootstrap() error {
return nil return nil
} }
// Allows you to run the underliying query in a transaction.
func (conn *DB) withTx(fn func(*sql.Tx) error) error {
tx, err := conn.Connection.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if err := fn(tx); err != nil {
return err
}
return tx.Commit()
}
func (conn *DB) Query(sql string, args ...any) (*sql.Rows, error) { func (conn *DB) Query(sql string, args ...any) (*sql.Rows, error) {
conn.log(DEBUG, "Executing query:", sql, args) conn.log(DEBUG, "Executing query:", sql, args)
@ -144,13 +163,68 @@ func (conn *DB) Query(sql string, args ...any) (*sql.Rows, error) {
return rows, nil return rows, nil
} }
// Creates a new Platform with the provided data.
//
// A collection is only a Node wrapper with some extended functionality to
// manage multiple nodes. For more information see [DB.NewNode] method and the
// [Platform] struct.
func (conn *DB) NewPlatform(data StandardNode) (*Platform, error) {
var platform *Platform
err := conn.withTx(func(tx *sql.Tx) error {
node, err := conn.newNodewithTx(tx, "PLATFORM", data)
if err != nil {
return err
}
collection, err := conn.newCollectionwithTx(
tx,
&default_collection{platform_name: node.name},
)
if err != nil {
return err
}
platform := &Platform{
Node: *node,
collections: []*Collection{collection},
}
_, err = conn.addRelationwithTx(tx, platform.Id, &collection_relation{}, collection.Id)
if err != nil {
return err
}
return nil
})
return platform, err
}
// Creates a new Collection with the provided data. // Creates a new Collection with the provided data.
// //
// A collection is only a Node wrapper with some extended functionality to // A collection is only a Node wrapper with some extended functionality to
// manage multiple nodes. For more information see [DB.NewNode] method and the // manage multiple nodes. For more information see [DB.NewNode] method and the
// [Collection] struct. // [Collection] struct.
func (conn *DB) NewCollection(data NodeClass) (*Collection, error) { //
node, err := conn.NewNode(data) // The operation is ran in a database transaction.
func (conn *DB) NewCollection(data StandardNode) (*Collection, error) {
var collection *Collection
err := conn.withTx(func(tx *sql.Tx) error {
var err error
collection, err = conn.newCollectionwithTx(tx, data)
return err
})
return collection, err
}
// Creates a new Collection with the provided data.
//
// A collection is only a Node wrapper with some extended functionality to
// manage multiple nodes. For more information see [DB.NewNode] method and the
// [Collection] struct.
func (conn *DB) newCollectionwithTx(tx *sql.Tx, data StandardNode) (*Collection, error) {
node, err := conn.newNodewithTx(tx, "COLLECTION", data)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -160,12 +234,27 @@ func (conn *DB) NewCollection(data NodeClass) (*Collection, error) {
childs: make([]*Node, 0), childs: make([]*Node, 0),
} }
return collection, nil return collection, err
}
// Creates a new node.
//
// The operation is ran in a database transaction.
func (conn *DB) NewNode(class string, data StandardNode) (*Node, error) {
var node *Node
err := conn.withTx(func(tx *sql.Tx) error {
var err error
node, err = conn.newNodewithTx(tx, class, data)
return err
})
return node, err
} }
// Creates a new node // Creates a new node
func (conn *DB) NewNode(data NodeClass) (*Node, error) { func (conn *DB) newNodewithTx(tx *sql.Tx, class string, data StandardNode) (*Node, error) {
class, name, metadata, err := data.ToNode() name, metadata, err := data.ToNode()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -178,13 +267,6 @@ func (conn *DB) NewNode(data NodeClass) (*Node, error) {
Id: -1, Id: -1,
} }
tx, err := conn.Connection.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
conn.log(DEBUG, "Creating node:", node) conn.log(DEBUG, "Creating node:", node)
sql := "INSERT INTO nodes (_class, name, metadata) VALUES ($1, $2, $3) RETURNING id;" sql := "INSERT INTO nodes (_class, name, metadata) VALUES ($1, $2, $3) RETURNING id;"
@ -194,22 +276,18 @@ func (conn *DB) NewNode(data NodeClass) (*Node, error) {
return nil, err return nil, err
} }
if err := tx.Commit(); err != nil {
return nil, err
}
return &node, nil return &node, nil
} }
// Updates a node with the provided id and data // Updates a node with the provided id and data
func (conn *DB) UpdateNode(id int64, data NodeClass) (*Node, error) { func (conn *DB) UpdateNode(id int64, data StandardNode) (*Node, error) {
class, name, metadata, err := data.ToNode() name, metadata, err := data.ToNode()
if err != nil { if err != nil {
return nil, err return nil, err
} }
node := Node{ node := Node{
_conn: conn, _conn: conn,
_class: class,
name: name, name: name,
metadata: metadata, metadata: metadata,
Id: id, Id: id,
@ -275,12 +353,32 @@ func (conn *DB) DeleteNode(id int64) error {
return nil return nil
} }
// Creates a new node.
//
// The operation is ran in a database transaction.
func (conn *DB) AddRelation(
from int64,
data StandardRelationship,
to int64,
) (*Relationship, error) {
var relationship *Relationship
err := conn.withTx(func(tx *sql.Tx) error {
var err error
relationship, err = conn.addRelationwithTx(tx, from, data, to)
return err
})
return relationship, err
}
// Creates a new relationship between two nodes. // Creates a new relationship between two nodes.
// //
// It returns the created relationship representation. // It returns the created relationship representation.
func (conn *DB) AddRelation( func (conn *DB) addRelationwithTx(
tx *sql.Tx,
from int64, from int64,
data RelationshipClass, data StandardRelationship,
to int64, to int64,
) (*Relationship, error) { ) (*Relationship, error) {
class, metadata, err := data.ToRelationship() class, metadata, err := data.ToRelationship()
@ -296,13 +394,6 @@ func (conn *DB) AddRelation(
To: to, To: to,
} }
tx, err := conn.Connection.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
conn.log(DEBUG, "Creating relationship:", from, relationship, to) conn.log(DEBUG, "Creating relationship:", from, relationship, to)
sql := "INSERT INTO relationships (_class, node_from, node_to, metadata) VALUES ($1, $2, $3, $4) RETURNING node_from, node_to;" sql := "INSERT INTO relationships (_class, node_from, node_to, metadata) VALUES ($1, $2, $3, $4) RETURNING node_from, node_to;"
@ -312,9 +403,6 @@ func (conn *DB) AddRelation(
return nil, err return nil, err
} }
if err := tx.Commit(); err != nil {
return nil, err
}
return &relationship, nil return &relationship, nil
} }