diff --git a/.gitignore b/.gitignore index c1c145d..cdcd809 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ pip-selfcheck.json secring.* +**/*.sql diff --git a/.justfile b/.justfile new file mode 100644 index 0000000..30e466e --- /dev/null +++ b/.justfile @@ -0,0 +1,2 @@ +run: + go run examples/usage.go diff --git a/.mise.toml b/.mise.toml new file mode 100644 index 0000000..b55f8c1 --- /dev/null +++ b/.mise.toml @@ -0,0 +1,2 @@ +[tools] +go = "latest" diff --git a/examples/mock_data/gleam.csv b/examples/mock_data/gleam.csv new file mode 100644 index 0000000..5881f16 --- /dev/null +++ b/examples/mock_data/gleam.csv @@ -0,0 +1,6 @@ +name,category,metadata +Mist,web framework,"{""github"":""rawhat/mist"",""package"":""mist"",""hex"":""mist""}" +Gleeunit,testing,"{""github"":""gleam-lang/gleeunit"",""package"":""gleeunit"",""hex"":""gleeunit""}" +Gleam_http,http,"{""github"":""gleam-lang/http"",""package"":""gleam_http"",""hex"":""gleam_http""}" +Gleam_json,serialization,"{""github"":""gleam-lang/json"",""package"":""gleam_json"",""hex"":""gleam_json""}" +Gleam_otp,concurrency,"{""github"":""gleam-lang/otp"",""package"":""gleam_otp"",""hex"":""gleam_otp""}" diff --git a/examples/mock_data/go.csv b/examples/mock_data/go.csv new file mode 100644 index 0000000..9eb30f4 --- /dev/null +++ b/examples/mock_data/go.csv @@ -0,0 +1,6 @@ +name,category,metadata +Gin,web framework,"{""website"":""https://gin-gonic.com"",""github"":""gin-gonic/gin"",""package"":""github.com/gin-gonic/gin""}" +Cobra,cli,"{""website"":""https://cobra.dev"",""github"":""spf13/cobra"",""package"":""github.com/spf13/cobra""}" +GORM,orm,"{""website"":""https://gorm.io"",""github"":""go-gorm/gorm"",""package"":""gorm.io/gorm""}" +Colly,web scraping,"{""website"":""http://go-colly.org"",""github"":""gocolly/colly"",""package"":""github.com/gocolly/colly""}" +gorilla/websocket,websockets,"{""github"":""gorilla/websocket"",""package"":""github.com/gorilla/websocket"",""protocol"":""ws""}" diff --git a/examples/mock_data/javascript.csv b/examples/mock_data/javascript.csv new file mode 100644 index 0000000..f7f9870 --- /dev/null +++ b/examples/mock_data/javascript.csv @@ -0,0 +1,6 @@ +name,category,metadata +React,ui framework,"{""website"":""https://reactjs.org"",""github"":""facebook/react"",""npm"":""react"",""type"":""library""}" +Express,web framework,"{""website"":""https://expressjs.com"",""github"":""expressjs/express"",""npm"":""express"",""type"":""framework""}" +Axios,http client,"{""website"":""https://axios-http.com"",""github"":""axios/axios"",""npm"":""axios"",""type"":""library""}" +Socket.io,websockets,"{""website"":""https://socket.io"",""github"":""socketio/socket.io"",""npm"":""socket.io"",""protocol"":""ws""}" +Commander.js,cli,"{""website"":""https://tj.github.io/commander.js"",""github"":""tj/commander.js"",""npm"":""commander""}" diff --git a/examples/mock_data/php.csv b/examples/mock_data/php.csv new file mode 100644 index 0000000..4db72b5 --- /dev/null +++ b/examples/mock_data/php.csv @@ -0,0 +1,6 @@ +name,category,metadata +Laravel,web framework,"{""website"":""https://laravel.com"",""github"":""laravel/laravel"",""packagist"":""laravel/framework""}" +Symfony,web framework,"{""website"":""https://symfony.com"",""github"":""symfony/symfony"",""packagist"":""symfony/symfony""}" +Guzzle,http client,"{""website"":""http://guzzlephp.org"",""github"":""guzzle/guzzle"",""packagist"":""guzzle/guzzle""}" +PHPUnit,testing,"{""website"":""https://phpunit.de"",""github"":""sebastianbergmann/phpunit"",""packagist"":""phpunit/phpunit""}" +Composer,package manager,"{""website"":""https://getcomposer.org"",""github"":""composer/composer"",""type"":""tool""}" diff --git a/examples/mock_data/python.csv b/examples/mock_data/python.csv new file mode 100644 index 0000000..90efc99 --- /dev/null +++ b/examples/mock_data/python.csv @@ -0,0 +1,6 @@ +name,category,metadata +Django,web framework,"{""website"":""https://djangoproject.com"",""github"":""django/django"",""package_manager"":""pip"",""latest_version"":""4.2.0""}" +FastAPI,web framework,"{""website"":""https://fastapi.tiangolo.com"",""github"":""tiangolo/fastapi"",""package_manager"":""pip"",""async_support"":true}" +Requests,http client,"{""website"":""https://requests.readthedocs.io"",""github"":""psf/requests"",""package_manager"":""pip"",""type"":""library""}" +Beautiful Soup,web scraping,"{""website"":""https://www.crummy.com/software/BeautifulSoup"",""github"":""waylan/beautifulsoup"",""parser_support"":[""html"",""xml""]}" +Click,cli,"{""website"":""https://click.palletsprojects.com"",""github"":""pallets/click"",""type"":""framework"",""python_versions"":"">=3.7""}" diff --git a/examples/mock_data/rust.csv b/examples/mock_data/rust.csv new file mode 100644 index 0000000..3dea97d --- /dev/null +++ b/examples/mock_data/rust.csv @@ -0,0 +1,6 @@ +name,category,metadata +Actix,web framework,"{""website"":""https://actix.rs"",""github"":""actix/actix-web"",""crate"":""actix-web""}" +Clap,cli,"{""website"":""https://clap.rs"",""github"":""clap-rs/clap"",""crate"":""clap""}" +Tokio,async runtime,"{""website"":""https://tokio.rs"",""github"":""tokio-rs/tokio"",""crate"":""tokio""}" +Reqwest,http client,"{""github"":""seanmonstar/reqwest"",""crate"":""reqwest"",""async_support"":true}" +Serde,serialization,"{""website"":""https://serde.rs"",""github"":""serde-rs/serde"",""crate"":""serde""}" diff --git a/examples/usage.go b/examples/usage.go new file mode 100644 index 0000000..264e5c4 --- /dev/null +++ b/examples/usage.go @@ -0,0 +1,222 @@ +package main + +import ( + "database/sql" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" + _ "modernc.org/sqlite" +) + +type ProgrammingLanguage struct { + Name string +} + +func (language *ProgrammingLanguage) ToNode() (string, string, []byte, error) { + metadata, err := json.Marshal("{\"test\": \"foo\"}") + if err != nil { + return "", "", nil, err + } + return "PROGRAMMING_LANGUAGE", language.Name, metadata, nil +} + +func (language *ProgrammingLanguage) FromNode(_class string, name string, metadata []byte) error { + if _class != "PROGRAMMING_LANGUAGE" { + return fmt.Errorf("invalid class %s", _class) + } + language.Name = name + return nil +} + +type Library struct { + Name string `json:"name"` + Category string `json:"category"` + Metadata map[string]interface{} `json:"metadata"` +} + +func (library *Library) ToNode() (string, string, []byte, error) { + metadata, err := json.Marshal(library.Metadata) + if err != nil { + return "", "", nil, err + } + return "LIBRARY", library.Name, metadata, nil +} + +func (library *Library) FromNode(_class string, name string, metadata []byte) error { + if _class != "LIBRARY" { + return fmt.Errorf("invalid class %s", _class) + } + if err := json.Unmarshal(metadata, &library.Metadata); err != nil { + return err + } + library.Name = name + return nil +} + +type ( + BelognsTo struct{} + IsSame struct{} +) + +func main() { + connection, err := sql.Open("sqlite", "db.sql") + if err != nil { + fmt.Println(err) + + return + } + + defer connection.Close() + + opts := synchronizator.DefaultOptions + // opts.Log_level = synchronizator.DEBUG + opts.DANGEROUSLY_DROP_TABLES = true + + sync, err := synchronizator.New(connection, opts) + if err != nil { + fmt.Println(err) + + return + } + + languages, err := loadData() + if err != nil { + fmt.Println(err) + } + + for language, libraries := range languages { + _, err := generateCollection( + &ProgrammingLanguage{Name: strings.ToUpper(language)}, + libraries, + sync, + ) + if err != nil { + println(err) + } + + // fmt.Fprintf( + // os.Stderr, + // "libraries_collection%+v\n", + // libraries_collection, + // ) + } + + golang, err := sync.GetNode(1) + if err != nil { + println(err) + } + fmt.Println("%v", golang) + relationships, err := golang.GetOutRelations() + if err != nil { + panic(err) + } + + for _, relationship := range relationships { + fmt.Printf("%v -> %v -> %v\n", relationship.From, relationship.GetClass(), relationship.To) + } +} + +// generateCollection Main example of the usage of the synchronizator package +func generateCollection( + language *ProgrammingLanguage, + libraries []Library, + sync *synchronizator.DB, +) (*synchronizator.Collection, error) { + language_libraries, err := sync.NewCollection(language) + if err != nil { + return nil, err + } + + for _, library := range libraries { + node, err := sync.NewNode(&library) + if err != nil { + return nil, err + } + data := &Library{} + if err := node.Unmarshall(data); err != nil { + println(err) + } + + if err := language_libraries.AddChild(node); err != nil { + return nil, err + } + } + + return language_libraries, nil +} + +func loadData() (map[string][]Library, error) { + // Find all CSV files + files, err := filepath.Glob("examples/mock_data/*.csv") + if err != nil { + return nil, fmt.Errorf("failed to glob files: %w", err) + } + + result := make(map[string][]Library) + + for _, file := range files { + // Load CSV file + libraries, err := processCSVFile(file) + if err != nil { + return nil, fmt.Errorf("failed to process %s: %w", file, err) + } + + // Use base filename without extension as language_name + language_name := filepath.Base(file) + language_name = language_name[:len(language_name)-len(filepath.Ext(language_name))] + + result[language_name] = libraries + } + + return result, nil +} + +func processCSVFile(filename string) ([]Library, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + reader := csv.NewReader(file) + + // Skip header + _, err = reader.Read() + if err != nil { + return nil, err + } + + var libraries []Library + + // Read records + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + // Parse metadata JSON + var metadata map[string]interface{} + if err := json.Unmarshal([]byte(record[2]), &metadata); err != nil { + return nil, fmt.Errorf("failed to parse metadata: %w", err) + } + + library := Library{ + Name: record[0], + Category: record[1], + Metadata: metadata, + } + libraries = append(libraries, library) + } + + return libraries, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6397324 --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module git.alecodes.page/alecodes/synchronizator + +go 1.23.2 + +require modernc.org/sqlite v1.33.1 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.22.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..617fda4 --- /dev/null +++ b/go.sum @@ -0,0 +1,49 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/pkg/collection.go b/pkg/collection.go new file mode 100644 index 0000000..957b951 --- /dev/null +++ b/pkg/collection.go @@ -0,0 +1,37 @@ +package synchronizator + +import "fmt" + +// Utility struct to represent a collection of nodes, it's a [Node] itself so all +// the node's functionality is available. +type Collection struct { + Node // Underlaying node info + childs []*Node // Child nodes +} + +// Internal RelationshipClass to handle the collection to node relationship +type collection_relation struct{} + +func (collection *collection_relation) ToRelationship() (string, []byte, error) { + return "COLLECTION_HAS", nil, nil +} + +func (collection *collection_relation) FromRelationship(_class string, metadata []byte) error { + if _class != "COLLECTION_HAS" { + return fmt.Errorf("invalid class %s", _class) + } + return nil +} + +// Adds a new child to this collection. Use the underlaying node's AddRelation +// method. +func (collection *Collection) AddChild(node *Node) error { + _, err := collection.AddRelation(&collection_relation{}, node.Id) + if err != nil { + return err + } + + collection.childs = append(collection.childs, node) + + return nil +} diff --git a/pkg/loglevel_string.go b/pkg/loglevel_string.go new file mode 100644 index 0000000..c27122d --- /dev/null +++ b/pkg/loglevel_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=LogLevel"; DO NOT EDIT. + +package synchronizator + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ERROR-0] + _ = x[INFO-1] + _ = x[DEBUG-2] +} + +const _LogLevel_name = "ERRORINFODEBUG" + +var _LogLevel_index = [...]uint8{0, 5, 9, 14} + +func (i LogLevel) String() string { + if i < 0 || i >= LogLevel(len(_LogLevel_index)-1) { + return "LogLevel(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _LogLevel_name[_LogLevel_index[i]:_LogLevel_index[i+1]] +} diff --git a/pkg/node.go b/pkg/node.go new file mode 100644 index 0000000..ae33c68 --- /dev/null +++ b/pkg/node.go @@ -0,0 +1,150 @@ +package synchronizator + +// A user representation of a node, this interface should be implemented by the user +// to provide the ability to parse the database node into a user defined +// struct that fulfills it's requirements. +// +// Example usage: +// +// type Library struct { +// Name string `json:"name"` +// Category string `json:"category"` +// Metadata map[string]interface{} `json:"metadata"` +// } +// +// func (library *Library) ToNode() (string, string, []byte, error) { +// metadata, err := json.Marshal(library.Metadata) +// if err != nil { +// return "", "", nil, err +// } +// return "LIBRARY", library.Name, metadata, nil +// } +// +// func (library *Library) FromNode(_class string, name string, metadata []byte) error { +// if _class != "LIBRARY" { +// return fmt.Errorf("invalid class %s", _class) +// } +// if err := json.Unmarshal(metadata, &library.Metadata); err != nil { +// return err +// } +// library.Name = name +// return nil +// } +type NodeClass interface { + // How to transform the struct into a node. It needs to return the class, + // name and a []byte representation of the metadata. + // + // - class: Is used for classification and query pourposes. It's recomended to provide a constante string to increase consistency. + // - name: A user friendly name + // - metadata: Arbitrary data. This will be stored as a jsonb in the database + // + ToNode() (string, string, []byte, error) + + // How to transform a node into the struct. This method should modify the + // struct directly as it receives a pointer. + // + // - class: Is used for classification and query pourposes. + // - name: A user friendly name + // - metadata: Arbitrary data. This is stored as a jsonb in the database + FromNode(string, string, []byte) error +} + +// A node in the database. +// It adds some helper methods to easily manage the node. +type Node struct { + _conn *DB // Underlaying connection to the database. + _class string // The class of the node, should not be modified to avoid inconsistencies. + _relationships []*Relationship // Relationships of the node + Id int64 // The id of the node + name string // The name of the node + metadata []byte // Arbitrary data. This is stored as a jsonb in the database +} + +// Creates a new relation of type RelationshipClass to the node with the +// provided id. An error is returned if the relation already exists. +// +// This method is a wrapper around the AddRelation method of the connection. +func (node *Node) AddRelation(relation RelationshipClass, to int64) (*Relationship, error) { + return node._conn.AddRelation(node.Id, relation, to) +} + +// Update a relation with the node of the provided id. +// +// This method is a wrapper around the UpdateRelation method of the connection. +func (node *Node) UpdateRelation(metadata any, to int64) error { + return node._conn.UpdateRelation(node.Id, metadata, to) +} + +// Delete a relation with the node of the provided id. +// +// This method is a wrapper around the DeleteRelation method of the connection. +func (node *Node) DeleteRelation(to int64) error { + return node._conn.DeleteRelation(node.Id, to) +} + +// Fetch all the outgoing relations for this node. This method will return a +// slice of relationships pointers and also store them in the node for further +// use. +func (node *Node) GetOutRelations() ([]*Relationship, error) { + sql := ` +WITH RECURSIVE NodeRelationships AS ( + SELECT + *, + relationships._class AS relationship_class, + relationships.metadata AS relationship_metadata + FROM + nodes as src + JOIN relationships ON src.id = relationships.node_from + WHERE src.id = $1 +) +SELECT + NodeRelationships.id as src_node, + NodeRelationships.relationship_class, + NodeRelationships.relationship_metadata, + dst.id as dst_id +FROM + NodeRelationships + JOIN nodes as dst ON dst.id = NodeRelationships.node_to + ` + + rows, err := node._conn.Query(sql, node.Id) + if err != nil { + return nil, err + } + + defer rows.Close() + + node._relationships = make([]*Relationship, 0) + for rows.Next() { + relationship := &Relationship{ + _conn: node._conn, + } + + if err := rows.Scan(&relationship.From, &relationship._class, &relationship.Metadata, &relationship.To); err != nil { + return nil, err + } + + node._relationships = append(node._relationships, relationship) + } + + return node._relationships, nil +} + +// Deletes this node from the database. +// This method is a wrapper around the DeleteNode method of the connection. +func (node *Node) Delete() error { + return node._conn.DeleteNode(node.Id) +} + +// Allows to retreive the saved information back into the user struct. This +// method will call the [NodeClass.FromNode] of the provided struct. +// +// Example: +// +// data := &Library{} +// if err := node.Unmarshall(data); err != nil { +// println(err) +// } +func (node *Node) Unmarshall(dst NodeClass) error { + return dst.FromNode(node._class, node.name, node.metadata) +} diff --git a/pkg/relationship.go b/pkg/relationship.go new file mode 100644 index 0000000..42372e8 --- /dev/null +++ b/pkg/relationship.go @@ -0,0 +1,50 @@ +package synchronizator + +// A user representation of a relationship, this interface should be implemented by the user +// to provide the ability to parse the database relationship into a user defined +// struct that fulfills it's requirements. +// +// Example usage: +// +// type collection_relation struct{} +// +// func (collection *collection_relation) ToRelationship() (string, []byte, error) { +// return "COLLECTION_HAS", nil, nil +// } +// +// func (collection *collection_relation) FromRelationship(_class string, metadata []byte) error { +// if _class != "COLLECTION_HAS" { +// return fmt.Errorf("invalid class %s", _class) +// } +// return nil +// } +type RelationshipClass interface { + // How to transform the struct into a collection. It needs to return the class, + // and a []byte representation of the metadata. + // + // - class: Is used for classification and query pourposes. It's recomended to provide a constante string to increase consistency. + // - metadata: Arbitrary data. This will be stored as a jsonb in the database + // + ToRelationship() (string, []byte, error) + + // How to transform a relationship into the struct. This method should modify the + // struct directly as it receives a pointer. + // + // - class: Is used for classification and query pourposes. + // - metadata: Arbitrary data. This is stored as a jsonb in the database + FromRelationship(string, []byte) error +} + +// A relationship in the database. +// It adds some helper methods to easily manage the node. +type Relationship struct { + _conn *DB // Underlaying connection to the database. + _class string // The class of the node, should not be modified to avoid inconsistencies. + From int64 // From what node this relation comes from + To int64 // To what node this relation goes to + Metadata []byte // Arbitrary data. This is stored as a jsonb in the database +} + +func (relationship *Relationship) GetClass() string { + return relationship._class +} diff --git a/pkg/synchronizator.go b/pkg/synchronizator.go new file mode 100644 index 0000000..b680427 --- /dev/null +++ b/pkg/synchronizator.go @@ -0,0 +1,368 @@ +// Package synchronizator: Provides a library to help synchronize data from +// different sources. +// +// It does so implementing a graph database representing the relation of the +// different entities of data called "nodes", helping you find, create or +// delete the "equivalent" entities in the different sources. +package synchronizator + +import ( + sql "database/sql" + "encoding/json" + "fmt" + "os" + "time" +) + +// DB Syncronization database interface +type DB struct { + Connection *sql.DB // underliying database connection + logger *os.File + log_level LogLevel + drop_tables bool +} + +// Options for the DB struct +type Options struct { + Logger *os.File // Where to log, defaults to os.Stdout + Log_level LogLevel // Log level, defaults to INFO + DANGEROUSLY_DROP_TABLES bool // Drop tables on initialization, defaults to false +} + +type LogLevel int + +// The log levels, lower levels take precedence +// +//go:generate stringer -type=LogLevel +const ( + ERROR LogLevel = iota + INFO + DEBUG +) + +var DefaultOptions = &Options{ + Logger: os.Stdout, + Log_level: INFO, + DANGEROUSLY_DROP_TABLES: false, +} + +// Constructor for the DB struct, it's dicourage to create the struct yourself! +// It receives a pointer to a sql.DB connection and an optional pointer to an Options struct. +// +// You need to initialize the connection yourself, this library does not handle that and only use the provided connection. +// This also alows to append this labrary tables to your already existing database. +func New(connection *sql.DB, options *Options) (*DB, error) { + if options == nil { + options = DefaultOptions + } + + conn := DB{ + Connection: connection, + logger: options.Logger, + log_level: options.Log_level, + drop_tables: options.DANGEROUSLY_DROP_TABLES, + } + + err := conn.bootstrap() + if err != nil { + return nil, err + } + + return &conn, nil +} + +// Write a log message to the logger. +// +// It receives a LogLevel and a variadic argument that is directly passed to fmt.Fprintln. +// The log is only written if the LogLevel is the same or lower than the configured value in the struct. +func (conn *DB) log(level LogLevel, args ...any) { + if level > conn.log_level { + return + } + + fmt.Fprintf(conn.logger, "[ %s - %-5s ]: ", time.Now().Format(time.DateTime), level.String()) + fmt.Fprintln(conn.logger, args...) +} + +// Initialize the database, creating the tables and indexes. +func (conn *DB) bootstrap() error { + conn.log(INFO, "Initializing database...") + + sql := "" + + if conn.drop_tables { + sql += ` + DROP TABLE IF EXISTS nodes; + DROP TABLE IF EXISTS relationships; + DROP INDEX IF EXISTS node_class; + DROP INDEX IF EXISTS relationships_class; + ` + } + + sql += ` + CREATE TABLE IF NOT EXISTS nodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + _class text NOT NULL, + name TEXT, + metadata jsonb DEFAULT '{}' + ); + + CREATE INDEX IF NOT EXISTS node_class on nodes (_class); + + + CREATE TABLE IF NOT EXISTS relationships ( + node_from INTEGER NOT NULL, + node_to INTEGER NOT NULL, + _class text NOT NULL, + metadata jsonb DEFAULT '{}', + PRIMARY KEY (node_from, node_to), + CHECK (node_from != node_to), + CONSTRAINT fk_node_from_relationships FOREIGN KEY (node_from) REFERENCES nodes(id), + CONSTRAINT fk_node_to_relationships FOREIGN KEY (node_to) REFERENCES nodes(id) + ); + + CREATE INDEX IF NOT EXISTS relationships_class on relationships (_class); + ` + + conn.log(DEBUG, sql) + _, err := conn.Connection.Exec(sql) + if err != nil { + return err + } + + return nil +} + +func (conn *DB) Query(sql string, args ...any) (*sql.Rows, error) { + conn.log(DEBUG, "Executing query:", sql, args) + + rows, err := conn.Connection.Query(sql, args...) + if err != nil { + return nil, err + } + + return rows, nil +} + +// Creates a new Collection with the provided data. +// +// A collection is only a Node wrapper with some extended functionality to +// manage multiple nodes. For more information see [DB.NewNode] method and the +// [Collection] struct. +func (conn *DB) NewCollection(data NodeClass) (*Collection, error) { + node, err := conn.NewNode(data) + if err != nil { + return nil, err + } + + collection := &Collection{ + Node: *node, + childs: make([]*Node, 0), + } + + return collection, nil +} + +// Creates a new node +func (conn *DB) NewNode(data NodeClass) (*Node, error) { + class, name, metadata, err := data.ToNode() + if err != nil { + return nil, err + } + + node := Node{ + _conn: conn, + _class: class, + name: name, + metadata: metadata, + Id: -1, + } + + tx, err := conn.Connection.Begin() + if err != nil { + return nil, err + } + + defer tx.Rollback() + + conn.log(DEBUG, "Creating node:", node) + + sql := "INSERT INTO nodes (_class, name, metadata) VALUES ($1, $2, $3) RETURNING id;" + + err = tx.QueryRow(sql, node._class, node.name, metadata).Scan(&node.Id) + if err != nil { + return nil, err + } + + if err := tx.Commit(); err != nil { + return nil, err + } + return &node, nil +} + +// Updates a node with the provided id and data +func (conn *DB) UpdateNode(id int64, data NodeClass) (*Node, error) { + class, name, metadata, err := data.ToNode() + if err != nil { + return nil, err + } + + node := Node{ + _conn: conn, + _class: class, + name: name, + metadata: metadata, + Id: id, + } + + tx, err := conn.Connection.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + conn.log(DEBUG, "Updating node:", id, node) + + sql := "UPDATE nodes SET _class = $1, name = $2, metadata = $3 WHERE id = $4;" + + _, err = tx.Exec(sql, node._class, node.name, node.metadata, id) + if err != nil { + return nil, err + } + + if err := tx.Commit(); err != nil { + return nil, err + } + + return &node, nil +} + +// Return a node with the provided id +func (conn *DB) GetNode(id int64) (*Node, error) { + node := Node{ + Id: id, + _conn: conn, + } + sql := "SELECT _class, name, metadata FROM nodes WHERE id = $1;" + conn.log(DEBUG, sql) + + err := conn.Connection.QueryRow(sql, id).Scan(&node._class, &node.name, &node.metadata) + if err != nil { + conn.log(DEBUG, err) + return nil, fmt.Errorf("No row matching id = %v", id) + } + + return &node, nil +} + +// Deletes a node with the provided id +func (conn *DB) DeleteNode(id int64) error { + tx, err := conn.Connection.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + conn.log(DEBUG, "Deleting node:", id) + sql := "DELETE FROM nodes WHERE id = $1;" + + _, err = tx.Exec(sql, id) + if err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + return nil +} + +// Creates a new relationship between two nodes. +// +// It returns the created relationship representation. +func (conn *DB) AddRelation( + from int64, + data RelationshipClass, + to int64, +) (*Relationship, error) { + class, metadata, err := data.ToRelationship() + if err != nil { + return nil, err + } + + relationship := Relationship{ + _conn: conn, + _class: class, + Metadata: metadata, + From: from, + To: to, + } + + tx, err := conn.Connection.Begin() + if err != nil { + return nil, err + } + + defer tx.Rollback() + + conn.log(DEBUG, "Creating relationship:", from, relationship, to) + + sql := "INSERT INTO relationships (_class, node_from, node_to, metadata) VALUES ($1, $2, $3, $4) RETURNING node_from, node_to;" + + _, err = tx.Exec(sql, relationship._class, relationship.From, relationship.To, metadata) + if err != nil { + return nil, err + } + + if err := tx.Commit(); err != nil { + return nil, err + } + return &relationship, nil +} + +// Updates a relationship for the provided id and data +func (conn *DB) UpdateRelation(from int64, metadata any, to int64) error { + if metadata == nil { + return fmt.Errorf("metadata cannot be nil") + } + json_metadata, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("invalid metadata format: %w", err) + } + tx, err := conn.Connection.Begin() + if err != nil { + return err + } + defer tx.Rollback() + conn.log(DEBUG, "Updating relationship:", from, metadata, to) + sql := "UPDATE relationships SET metadata = $1 WHERE node_from = $2 AND node_to = $3;" + _, err = tx.Exec(sql, json_metadata, from, to) + if err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + return nil +} + +// Deletes a relationship for the provided id +func (conn *DB) DeleteRelation(from int64, to int64) error { + tx, err := conn.Connection.Begin() + if err != nil { + return err + } + + defer tx.Rollback() + + sql := "DELETE FROM relationships WHERE node_from = $1 AND node_to = $2;" + conn.log(DEBUG, "Deleting relationship:", from, to) + _, err = tx.Exec(sql, from, to) + if err != nil { + return nil + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +}