generated from alecodes/base-template
feat: implement database handlerImplement a database api to manage nodes and relationships in a "graph databaselike" implemented in sqlite.closes #1
This commit is contained in:
parent
5aaacb10e3
commit
b2d8dadcee
17 changed files with 963 additions and 0 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -16,3 +16,4 @@ pip-selfcheck.json
|
|||
secring.*
|
||||
|
||||
|
||||
**/*.sql
|
||||
|
|
|
|||
2
.justfile
Normal file
2
.justfile
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
run:
|
||||
go run examples/usage.go
|
||||
2
.mise.toml
Normal file
2
.mise.toml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
[tools]
|
||||
go = "latest"
|
||||
6
examples/mock_data/gleam.csv
Normal file
6
examples/mock_data/gleam.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
name,category,metadata
|
||||
Mist,web framework,"{""github"":""rawhat/mist"",""package"":""mist"",""hex"":""mist""}"
|
||||
Gleeunit,testing,"{""github"":""gleam-lang/gleeunit"",""package"":""gleeunit"",""hex"":""gleeunit""}"
|
||||
Gleam_http,http,"{""github"":""gleam-lang/http"",""package"":""gleam_http"",""hex"":""gleam_http""}"
|
||||
Gleam_json,serialization,"{""github"":""gleam-lang/json"",""package"":""gleam_json"",""hex"":""gleam_json""}"
|
||||
Gleam_otp,concurrency,"{""github"":""gleam-lang/otp"",""package"":""gleam_otp"",""hex"":""gleam_otp""}"
|
||||
|
6
examples/mock_data/go.csv
Normal file
6
examples/mock_data/go.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
name,category,metadata
|
||||
Gin,web framework,"{""website"":""https://gin-gonic.com"",""github"":""gin-gonic/gin"",""package"":""github.com/gin-gonic/gin""}"
|
||||
Cobra,cli,"{""website"":""https://cobra.dev"",""github"":""spf13/cobra"",""package"":""github.com/spf13/cobra""}"
|
||||
GORM,orm,"{""website"":""https://gorm.io"",""github"":""go-gorm/gorm"",""package"":""gorm.io/gorm""}"
|
||||
Colly,web scraping,"{""website"":""http://go-colly.org"",""github"":""gocolly/colly"",""package"":""github.com/gocolly/colly""}"
|
||||
gorilla/websocket,websockets,"{""github"":""gorilla/websocket"",""package"":""github.com/gorilla/websocket"",""protocol"":""ws""}"
|
||||
|
6
examples/mock_data/javascript.csv
Normal file
6
examples/mock_data/javascript.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
name,category,metadata
|
||||
React,ui framework,"{""website"":""https://reactjs.org"",""github"":""facebook/react"",""npm"":""react"",""type"":""library""}"
|
||||
Express,web framework,"{""website"":""https://expressjs.com"",""github"":""expressjs/express"",""npm"":""express"",""type"":""framework""}"
|
||||
Axios,http client,"{""website"":""https://axios-http.com"",""github"":""axios/axios"",""npm"":""axios"",""type"":""library""}"
|
||||
Socket.io,websockets,"{""website"":""https://socket.io"",""github"":""socketio/socket.io"",""npm"":""socket.io"",""protocol"":""ws""}"
|
||||
Commander.js,cli,"{""website"":""https://tj.github.io/commander.js"",""github"":""tj/commander.js"",""npm"":""commander""}"
|
||||
|
6
examples/mock_data/php.csv
Normal file
6
examples/mock_data/php.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
name,category,metadata
|
||||
Laravel,web framework,"{""website"":""https://laravel.com"",""github"":""laravel/laravel"",""packagist"":""laravel/framework""}"
|
||||
Symfony,web framework,"{""website"":""https://symfony.com"",""github"":""symfony/symfony"",""packagist"":""symfony/symfony""}"
|
||||
Guzzle,http client,"{""website"":""http://guzzlephp.org"",""github"":""guzzle/guzzle"",""packagist"":""guzzle/guzzle""}"
|
||||
PHPUnit,testing,"{""website"":""https://phpunit.de"",""github"":""sebastianbergmann/phpunit"",""packagist"":""phpunit/phpunit""}"
|
||||
Composer,package manager,"{""website"":""https://getcomposer.org"",""github"":""composer/composer"",""type"":""tool""}"
|
||||
|
6
examples/mock_data/python.csv
Normal file
6
examples/mock_data/python.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
name,category,metadata
|
||||
Django,web framework,"{""website"":""https://djangoproject.com"",""github"":""django/django"",""package_manager"":""pip"",""latest_version"":""4.2.0""}"
|
||||
FastAPI,web framework,"{""website"":""https://fastapi.tiangolo.com"",""github"":""tiangolo/fastapi"",""package_manager"":""pip"",""async_support"":true}"
|
||||
Requests,http client,"{""website"":""https://requests.readthedocs.io"",""github"":""psf/requests"",""package_manager"":""pip"",""type"":""library""}"
|
||||
Beautiful Soup,web scraping,"{""website"":""https://www.crummy.com/software/BeautifulSoup"",""github"":""waylan/beautifulsoup"",""parser_support"":[""html"",""xml""]}"
|
||||
Click,cli,"{""website"":""https://click.palletsprojects.com"",""github"":""pallets/click"",""type"":""framework"",""python_versions"":"">=3.7""}"
|
||||
|
6
examples/mock_data/rust.csv
Normal file
6
examples/mock_data/rust.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
name,category,metadata
|
||||
Actix,web framework,"{""website"":""https://actix.rs"",""github"":""actix/actix-web"",""crate"":""actix-web""}"
|
||||
Clap,cli,"{""website"":""https://clap.rs"",""github"":""clap-rs/clap"",""crate"":""clap""}"
|
||||
Tokio,async runtime,"{""website"":""https://tokio.rs"",""github"":""tokio-rs/tokio"",""crate"":""tokio""}"
|
||||
Reqwest,http client,"{""github"":""seanmonstar/reqwest"",""crate"":""reqwest"",""async_support"":true}"
|
||||
Serde,serialization,"{""website"":""https://serde.rs"",""github"":""serde-rs/serde"",""crate"":""serde""}"
|
||||
|
222
examples/usage.go
Normal file
222
examples/usage.go
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type ProgrammingLanguage struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
func (language *ProgrammingLanguage) ToNode() (string, string, []byte, error) {
|
||||
metadata, err := json.Marshal("{\"test\": \"foo\"}")
|
||||
if err != nil {
|
||||
return "", "", nil, err
|
||||
}
|
||||
return "PROGRAMMING_LANGUAGE", language.Name, metadata, nil
|
||||
}
|
||||
|
||||
func (language *ProgrammingLanguage) FromNode(_class string, name string, metadata []byte) error {
|
||||
if _class != "PROGRAMMING_LANGUAGE" {
|
||||
return fmt.Errorf("invalid class %s", _class)
|
||||
}
|
||||
language.Name = name
|
||||
return nil
|
||||
}
|
||||
|
||||
type Library struct {
|
||||
Name string `json:"name"`
|
||||
Category string `json:"category"`
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
}
|
||||
|
||||
func (library *Library) ToNode() (string, string, []byte, error) {
|
||||
metadata, err := json.Marshal(library.Metadata)
|
||||
if err != nil {
|
||||
return "", "", nil, err
|
||||
}
|
||||
return "LIBRARY", library.Name, metadata, nil
|
||||
}
|
||||
|
||||
func (library *Library) FromNode(_class string, name string, metadata []byte) error {
|
||||
if _class != "LIBRARY" {
|
||||
return fmt.Errorf("invalid class %s", _class)
|
||||
}
|
||||
if err := json.Unmarshal(metadata, &library.Metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
library.Name = name
|
||||
return nil
|
||||
}
|
||||
|
||||
type (
|
||||
BelognsTo struct{}
|
||||
IsSame struct{}
|
||||
)
|
||||
|
||||
func main() {
|
||||
connection, err := sql.Open("sqlite", "db.sql")
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer connection.Close()
|
||||
|
||||
opts := synchronizator.DefaultOptions
|
||||
// opts.Log_level = synchronizator.DEBUG
|
||||
opts.DANGEROUSLY_DROP_TABLES = true
|
||||
|
||||
sync, err := synchronizator.New(connection, opts)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
languages, err := loadData()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
for language, libraries := range languages {
|
||||
_, err := generateCollection(
|
||||
&ProgrammingLanguage{Name: strings.ToUpper(language)},
|
||||
libraries,
|
||||
sync,
|
||||
)
|
||||
if err != nil {
|
||||
println(err)
|
||||
}
|
||||
|
||||
// fmt.Fprintf(
|
||||
// os.Stderr,
|
||||
// "libraries_collection%+v\n",
|
||||
// libraries_collection,
|
||||
// )
|
||||
}
|
||||
|
||||
golang, err := sync.GetNode(1)
|
||||
if err != nil {
|
||||
println(err)
|
||||
}
|
||||
fmt.Println("%v", golang)
|
||||
relationships, err := golang.GetOutRelations()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, relationship := range relationships {
|
||||
fmt.Printf("%v -> %v -> %v\n", relationship.From, relationship.GetClass(), relationship.To)
|
||||
}
|
||||
}
|
||||
|
||||
// generateCollection Main example of the usage of the synchronizator package
|
||||
func generateCollection(
|
||||
language *ProgrammingLanguage,
|
||||
libraries []Library,
|
||||
sync *synchronizator.DB,
|
||||
) (*synchronizator.Collection, error) {
|
||||
language_libraries, err := sync.NewCollection(language)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, library := range libraries {
|
||||
node, err := sync.NewNode(&library)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := &Library{}
|
||||
if err := node.Unmarshall(data); err != nil {
|
||||
println(err)
|
||||
}
|
||||
|
||||
if err := language_libraries.AddChild(node); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return language_libraries, nil
|
||||
}
|
||||
|
||||
func loadData() (map[string][]Library, error) {
|
||||
// Find all CSV files
|
||||
files, err := filepath.Glob("examples/mock_data/*.csv")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to glob files: %w", err)
|
||||
}
|
||||
|
||||
result := make(map[string][]Library)
|
||||
|
||||
for _, file := range files {
|
||||
// Load CSV file
|
||||
libraries, err := processCSVFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to process %s: %w", file, err)
|
||||
}
|
||||
|
||||
// Use base filename without extension as language_name
|
||||
language_name := filepath.Base(file)
|
||||
language_name = language_name[:len(language_name)-len(filepath.Ext(language_name))]
|
||||
|
||||
result[language_name] = libraries
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func processCSVFile(filename string) ([]Library, error) {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := csv.NewReader(file)
|
||||
|
||||
// Skip header
|
||||
_, err = reader.Read()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var libraries []Library
|
||||
|
||||
// Read records
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
var metadata map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(record[2]), &metadata); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse metadata: %w", err)
|
||||
}
|
||||
|
||||
library := Library{
|
||||
Name: record[0],
|
||||
Category: record[1],
|
||||
Metadata: metadata,
|
||||
}
|
||||
libraries = append(libraries, library)
|
||||
}
|
||||
|
||||
return libraries, nil
|
||||
}
|
||||
21
go.mod
Normal file
21
go.mod
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
module git.alecodes.page/alecodes/synchronizator
|
||||
|
||||
go 1.23.2
|
||||
|
||||
require modernc.org/sqlite v1.33.1
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.8.0 // indirect
|
||||
modernc.org/strutil v1.2.0 // indirect
|
||||
modernc.org/token v1.1.0 // indirect
|
||||
)
|
||||
49
go.sum
Normal file
49
go.sum
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
|
||||
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
|
||||
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
|
||||
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
|
||||
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
|
||||
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
|
||||
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
|
||||
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
|
||||
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
|
||||
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
|
||||
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
|
||||
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
|
||||
modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
|
||||
modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
|
||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
37
pkg/collection.go
Normal file
37
pkg/collection.go
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
package synchronizator
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Utility struct to represent a collection of nodes, it's a [Node] itself so all
|
||||
// the node's functionality is available.
|
||||
type Collection struct {
|
||||
Node // Underlaying node info
|
||||
childs []*Node // Child nodes
|
||||
}
|
||||
|
||||
// Internal RelationshipClass to handle the collection to node relationship
|
||||
type collection_relation struct{}
|
||||
|
||||
func (collection *collection_relation) ToRelationship() (string, []byte, error) {
|
||||
return "COLLECTION_HAS", nil, nil
|
||||
}
|
||||
|
||||
func (collection *collection_relation) FromRelationship(_class string, metadata []byte) error {
|
||||
if _class != "COLLECTION_HAS" {
|
||||
return fmt.Errorf("invalid class %s", _class)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Adds a new child to this collection. Use the underlaying node's AddRelation
|
||||
// method.
|
||||
func (collection *Collection) AddChild(node *Node) error {
|
||||
_, err := collection.AddRelation(&collection_relation{}, node.Id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
collection.childs = append(collection.childs, node)
|
||||
|
||||
return nil
|
||||
}
|
||||
25
pkg/loglevel_string.go
Normal file
25
pkg/loglevel_string.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
// Code generated by "stringer -type=LogLevel"; DO NOT EDIT.
|
||||
|
||||
package synchronizator
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[ERROR-0]
|
||||
_ = x[INFO-1]
|
||||
_ = x[DEBUG-2]
|
||||
}
|
||||
|
||||
const _LogLevel_name = "ERRORINFODEBUG"
|
||||
|
||||
var _LogLevel_index = [...]uint8{0, 5, 9, 14}
|
||||
|
||||
func (i LogLevel) String() string {
|
||||
if i < 0 || i >= LogLevel(len(_LogLevel_index)-1) {
|
||||
return "LogLevel(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _LogLevel_name[_LogLevel_index[i]:_LogLevel_index[i+1]]
|
||||
}
|
||||
150
pkg/node.go
Normal file
150
pkg/node.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
package synchronizator
|
||||
|
||||
// A user representation of a node, this interface should be implemented by the user
|
||||
// to provide the ability to parse the database node into a user defined
|
||||
// struct that fulfills it's requirements.
|
||||
//
|
||||
// Example usage:
|
||||
//
|
||||
// type Library struct {
|
||||
// Name string `json:"name"`
|
||||
// Category string `json:"category"`
|
||||
// Metadata map[string]interface{} `json:"metadata"`
|
||||
// }
|
||||
//
|
||||
// func (library *Library) ToNode() (string, string, []byte, error) {
|
||||
// metadata, err := json.Marshal(library.Metadata)
|
||||
// if err != nil {
|
||||
// return "", "", nil, err
|
||||
// }
|
||||
// return "LIBRARY", library.Name, metadata, nil
|
||||
// }
|
||||
//
|
||||
// func (library *Library) FromNode(_class string, name string, metadata []byte) error {
|
||||
// if _class != "LIBRARY" {
|
||||
// return fmt.Errorf("invalid class %s", _class)
|
||||
// }
|
||||
// if err := json.Unmarshal(metadata, &library.Metadata); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// library.Name = name
|
||||
// return nil
|
||||
// }
|
||||
type NodeClass interface {
|
||||
// How to transform the struct into a node. It needs to return the class,
|
||||
// name and a []byte representation of the metadata.
|
||||
//
|
||||
// - class: Is used for classification and query pourposes. It's recomended to provide a constante string to increase consistency.
|
||||
// - name: A user friendly name
|
||||
// - metadata: Arbitrary data. This will be stored as a jsonb in the database
|
||||
//
|
||||
ToNode() (string, string, []byte, error)
|
||||
|
||||
// How to transform a node into the struct. This method should modify the
|
||||
// struct directly as it receives a pointer.
|
||||
//
|
||||
// - class: Is used for classification and query pourposes.
|
||||
// - name: A user friendly name
|
||||
// - metadata: Arbitrary data. This is stored as a jsonb in the database
|
||||
FromNode(string, string, []byte) error
|
||||
}
|
||||
|
||||
// A node in the database.
|
||||
// It adds some helper methods to easily manage the node.
|
||||
type Node struct {
|
||||
_conn *DB // Underlaying connection to the database.
|
||||
_class string // The class of the node, should not be modified to avoid inconsistencies.
|
||||
_relationships []*Relationship // Relationships of the node
|
||||
Id int64 // The id of the node
|
||||
name string // The name of the node
|
||||
metadata []byte // Arbitrary data. This is stored as a jsonb in the database
|
||||
}
|
||||
|
||||
// Creates a new relation of type RelationshipClass to the node with the
|
||||
// provided id. An error is returned if the relation already exists.
|
||||
//
|
||||
// This method is a wrapper around the AddRelation method of the connection.
|
||||
func (node *Node) AddRelation(relation RelationshipClass, to int64) (*Relationship, error) {
|
||||
return node._conn.AddRelation(node.Id, relation, to)
|
||||
}
|
||||
|
||||
// Update a relation with the node of the provided id.
|
||||
//
|
||||
// This method is a wrapper around the UpdateRelation method of the connection.
|
||||
func (node *Node) UpdateRelation(metadata any, to int64) error {
|
||||
return node._conn.UpdateRelation(node.Id, metadata, to)
|
||||
}
|
||||
|
||||
// Delete a relation with the node of the provided id.
|
||||
//
|
||||
// This method is a wrapper around the DeleteRelation method of the connection.
|
||||
func (node *Node) DeleteRelation(to int64) error {
|
||||
return node._conn.DeleteRelation(node.Id, to)
|
||||
}
|
||||
|
||||
// Fetch all the outgoing relations for this node. This method will return a
|
||||
// slice of relationships pointers and also store them in the node for further
|
||||
// use.
|
||||
func (node *Node) GetOutRelations() ([]*Relationship, error) {
|
||||
sql := `
|
||||
WITH RECURSIVE NodeRelationships AS (
|
||||
SELECT
|
||||
*,
|
||||
relationships._class AS relationship_class,
|
||||
relationships.metadata AS relationship_metadata
|
||||
FROM
|
||||
nodes as src
|
||||
JOIN relationships ON src.id = relationships.node_from
|
||||
WHERE src.id = $1
|
||||
)
|
||||
SELECT
|
||||
NodeRelationships.id as src_node,
|
||||
NodeRelationships.relationship_class,
|
||||
NodeRelationships.relationship_metadata,
|
||||
dst.id as dst_id
|
||||
FROM
|
||||
NodeRelationships
|
||||
JOIN nodes as dst ON dst.id = NodeRelationships.node_to
|
||||
`
|
||||
|
||||
rows, err := node._conn.Query(sql, node.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer rows.Close()
|
||||
|
||||
node._relationships = make([]*Relationship, 0)
|
||||
for rows.Next() {
|
||||
relationship := &Relationship{
|
||||
_conn: node._conn,
|
||||
}
|
||||
|
||||
if err := rows.Scan(&relationship.From, &relationship._class, &relationship.Metadata, &relationship.To); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
node._relationships = append(node._relationships, relationship)
|
||||
}
|
||||
|
||||
return node._relationships, nil
|
||||
}
|
||||
|
||||
// Deletes this node from the database.
|
||||
// This method is a wrapper around the DeleteNode method of the connection.
|
||||
func (node *Node) Delete() error {
|
||||
return node._conn.DeleteNode(node.Id)
|
||||
}
|
||||
|
||||
// Allows to retreive the saved information back into the user struct. This
|
||||
// method will call the [NodeClass.FromNode] of the provided struct.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// data := &Library{}
|
||||
// if err := node.Unmarshall(data); err != nil {
|
||||
// println(err)
|
||||
// }
|
||||
func (node *Node) Unmarshall(dst NodeClass) error {
|
||||
return dst.FromNode(node._class, node.name, node.metadata)
|
||||
}
|
||||
50
pkg/relationship.go
Normal file
50
pkg/relationship.go
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
package synchronizator
|
||||
|
||||
// A user representation of a relationship, this interface should be implemented by the user
|
||||
// to provide the ability to parse the database relationship into a user defined
|
||||
// struct that fulfills it's requirements.
|
||||
//
|
||||
// Example usage:
|
||||
//
|
||||
// type collection_relation struct{}
|
||||
//
|
||||
// func (collection *collection_relation) ToRelationship() (string, []byte, error) {
|
||||
// return "COLLECTION_HAS", nil, nil
|
||||
// }
|
||||
//
|
||||
// func (collection *collection_relation) FromRelationship(_class string, metadata []byte) error {
|
||||
// if _class != "COLLECTION_HAS" {
|
||||
// return fmt.Errorf("invalid class %s", _class)
|
||||
// }
|
||||
// return nil
|
||||
// }
|
||||
type RelationshipClass interface {
|
||||
// How to transform the struct into a collection. It needs to return the class,
|
||||
// and a []byte representation of the metadata.
|
||||
//
|
||||
// - class: Is used for classification and query pourposes. It's recomended to provide a constante string to increase consistency.
|
||||
// - metadata: Arbitrary data. This will be stored as a jsonb in the database
|
||||
//
|
||||
ToRelationship() (string, []byte, error)
|
||||
|
||||
// How to transform a relationship into the struct. This method should modify the
|
||||
// struct directly as it receives a pointer.
|
||||
//
|
||||
// - class: Is used for classification and query pourposes.
|
||||
// - metadata: Arbitrary data. This is stored as a jsonb in the database
|
||||
FromRelationship(string, []byte) error
|
||||
}
|
||||
|
||||
// A relationship in the database.
|
||||
// It adds some helper methods to easily manage the node.
|
||||
type Relationship struct {
|
||||
_conn *DB // Underlaying connection to the database.
|
||||
_class string // The class of the node, should not be modified to avoid inconsistencies.
|
||||
From int64 // From what node this relation comes from
|
||||
To int64 // To what node this relation goes to
|
||||
Metadata []byte // Arbitrary data. This is stored as a jsonb in the database
|
||||
}
|
||||
|
||||
func (relationship *Relationship) GetClass() string {
|
||||
return relationship._class
|
||||
}
|
||||
368
pkg/synchronizator.go
Normal file
368
pkg/synchronizator.go
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
// Package synchronizator: Provides a library to help synchronize data from
|
||||
// different sources.
|
||||
//
|
||||
// It does so implementing a graph database representing the relation of the
|
||||
// different entities of data called "nodes", helping you find, create or
|
||||
// delete the "equivalent" entities in the different sources.
|
||||
package synchronizator
|
||||
|
||||
import (
|
||||
sql "database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DB Syncronization database interface
|
||||
type DB struct {
|
||||
Connection *sql.DB // underliying database connection
|
||||
logger *os.File
|
||||
log_level LogLevel
|
||||
drop_tables bool
|
||||
}
|
||||
|
||||
// Options for the DB struct
|
||||
type Options struct {
|
||||
Logger *os.File // Where to log, defaults to os.Stdout
|
||||
Log_level LogLevel // Log level, defaults to INFO
|
||||
DANGEROUSLY_DROP_TABLES bool // Drop tables on initialization, defaults to false
|
||||
}
|
||||
|
||||
type LogLevel int
|
||||
|
||||
// The log levels, lower levels take precedence
|
||||
//
|
||||
//go:generate stringer -type=LogLevel
|
||||
const (
|
||||
ERROR LogLevel = iota
|
||||
INFO
|
||||
DEBUG
|
||||
)
|
||||
|
||||
var DefaultOptions = &Options{
|
||||
Logger: os.Stdout,
|
||||
Log_level: INFO,
|
||||
DANGEROUSLY_DROP_TABLES: false,
|
||||
}
|
||||
|
||||
// Constructor for the DB struct, it's dicourage to create the struct yourself!
|
||||
// It receives a pointer to a sql.DB connection and an optional pointer to an Options struct.
|
||||
//
|
||||
// You need to initialize the connection yourself, this library does not handle that and only use the provided connection.
|
||||
// This also alows to append this labrary tables to your already existing database.
|
||||
func New(connection *sql.DB, options *Options) (*DB, error) {
|
||||
if options == nil {
|
||||
options = DefaultOptions
|
||||
}
|
||||
|
||||
conn := DB{
|
||||
Connection: connection,
|
||||
logger: options.Logger,
|
||||
log_level: options.Log_level,
|
||||
drop_tables: options.DANGEROUSLY_DROP_TABLES,
|
||||
}
|
||||
|
||||
err := conn.bootstrap()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &conn, nil
|
||||
}
|
||||
|
||||
// Write a log message to the logger.
|
||||
//
|
||||
// It receives a LogLevel and a variadic argument that is directly passed to fmt.Fprintln.
|
||||
// The log is only written if the LogLevel is the same or lower than the configured value in the struct.
|
||||
func (conn *DB) log(level LogLevel, args ...any) {
|
||||
if level > conn.log_level {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(conn.logger, "[ %s - %-5s ]: ", time.Now().Format(time.DateTime), level.String())
|
||||
fmt.Fprintln(conn.logger, args...)
|
||||
}
|
||||
|
||||
// Initialize the database, creating the tables and indexes.
|
||||
func (conn *DB) bootstrap() error {
|
||||
conn.log(INFO, "Initializing database...")
|
||||
|
||||
sql := ""
|
||||
|
||||
if conn.drop_tables {
|
||||
sql += `
|
||||
DROP TABLE IF EXISTS nodes;
|
||||
DROP TABLE IF EXISTS relationships;
|
||||
DROP INDEX IF EXISTS node_class;
|
||||
DROP INDEX IF EXISTS relationships_class;
|
||||
`
|
||||
}
|
||||
|
||||
sql += `
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
_class text NOT NULL,
|
||||
name TEXT,
|
||||
metadata jsonb DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS node_class on nodes (_class);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relationships (
|
||||
node_from INTEGER NOT NULL,
|
||||
node_to INTEGER NOT NULL,
|
||||
_class text NOT NULL,
|
||||
metadata jsonb DEFAULT '{}',
|
||||
PRIMARY KEY (node_from, node_to),
|
||||
CHECK (node_from != node_to),
|
||||
CONSTRAINT fk_node_from_relationships FOREIGN KEY (node_from) REFERENCES nodes(id),
|
||||
CONSTRAINT fk_node_to_relationships FOREIGN KEY (node_to) REFERENCES nodes(id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS relationships_class on relationships (_class);
|
||||
`
|
||||
|
||||
conn.log(DEBUG, sql)
|
||||
_, err := conn.Connection.Exec(sql)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (conn *DB) Query(sql string, args ...any) (*sql.Rows, error) {
|
||||
conn.log(DEBUG, "Executing query:", sql, args)
|
||||
|
||||
rows, err := conn.Connection.Query(sql, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// Creates a new Collection with the provided data.
|
||||
//
|
||||
// A collection is only a Node wrapper with some extended functionality to
|
||||
// manage multiple nodes. For more information see [DB.NewNode] method and the
|
||||
// [Collection] struct.
|
||||
func (conn *DB) NewCollection(data NodeClass) (*Collection, error) {
|
||||
node, err := conn.NewNode(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
collection := &Collection{
|
||||
Node: *node,
|
||||
childs: make([]*Node, 0),
|
||||
}
|
||||
|
||||
return collection, nil
|
||||
}
|
||||
|
||||
// Creates a new node
|
||||
func (conn *DB) NewNode(data NodeClass) (*Node, error) {
|
||||
class, name, metadata, err := data.ToNode()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
node := Node{
|
||||
_conn: conn,
|
||||
_class: class,
|
||||
name: name,
|
||||
metadata: metadata,
|
||||
Id: -1,
|
||||
}
|
||||
|
||||
tx, err := conn.Connection.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
|
||||
conn.log(DEBUG, "Creating node:", node)
|
||||
|
||||
sql := "INSERT INTO nodes (_class, name, metadata) VALUES ($1, $2, $3) RETURNING id;"
|
||||
|
||||
err = tx.QueryRow(sql, node._class, node.name, metadata).Scan(&node.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// Updates a node with the provided id and data
|
||||
func (conn *DB) UpdateNode(id int64, data NodeClass) (*Node, error) {
|
||||
class, name, metadata, err := data.ToNode()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
node := Node{
|
||||
_conn: conn,
|
||||
_class: class,
|
||||
name: name,
|
||||
metadata: metadata,
|
||||
Id: id,
|
||||
}
|
||||
|
||||
tx, err := conn.Connection.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
conn.log(DEBUG, "Updating node:", id, node)
|
||||
|
||||
sql := "UPDATE nodes SET _class = $1, name = $2, metadata = $3 WHERE id = $4;"
|
||||
|
||||
_, err = tx.Exec(sql, node._class, node.name, node.metadata, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// Return a node with the provided id
|
||||
func (conn *DB) GetNode(id int64) (*Node, error) {
|
||||
node := Node{
|
||||
Id: id,
|
||||
_conn: conn,
|
||||
}
|
||||
sql := "SELECT _class, name, metadata FROM nodes WHERE id = $1;"
|
||||
conn.log(DEBUG, sql)
|
||||
|
||||
err := conn.Connection.QueryRow(sql, id).Scan(&node._class, &node.name, &node.metadata)
|
||||
if err != nil {
|
||||
conn.log(DEBUG, err)
|
||||
return nil, fmt.Errorf("No row matching id = %v", id)
|
||||
}
|
||||
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// Deletes a node with the provided id
|
||||
func (conn *DB) DeleteNode(id int64) error {
|
||||
tx, err := conn.Connection.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
conn.log(DEBUG, "Deleting node:", id)
|
||||
sql := "DELETE FROM nodes WHERE id = $1;"
|
||||
|
||||
_, err = tx.Exec(sql, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Creates a new relationship between two nodes.
|
||||
//
|
||||
// It returns the created relationship representation.
|
||||
func (conn *DB) AddRelation(
|
||||
from int64,
|
||||
data RelationshipClass,
|
||||
to int64,
|
||||
) (*Relationship, error) {
|
||||
class, metadata, err := data.ToRelationship()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
relationship := Relationship{
|
||||
_conn: conn,
|
||||
_class: class,
|
||||
Metadata: metadata,
|
||||
From: from,
|
||||
To: to,
|
||||
}
|
||||
|
||||
tx, err := conn.Connection.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
|
||||
conn.log(DEBUG, "Creating relationship:", from, relationship, to)
|
||||
|
||||
sql := "INSERT INTO relationships (_class, node_from, node_to, metadata) VALUES ($1, $2, $3, $4) RETURNING node_from, node_to;"
|
||||
|
||||
_, err = tx.Exec(sql, relationship._class, relationship.From, relationship.To, metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &relationship, nil
|
||||
}
|
||||
|
||||
// Updates a relationship for the provided id and data
|
||||
func (conn *DB) UpdateRelation(from int64, metadata any, to int64) error {
|
||||
if metadata == nil {
|
||||
return fmt.Errorf("metadata cannot be nil")
|
||||
}
|
||||
json_metadata, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid metadata format: %w", err)
|
||||
}
|
||||
tx, err := conn.Connection.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
conn.log(DEBUG, "Updating relationship:", from, metadata, to)
|
||||
sql := "UPDATE relationships SET metadata = $1 WHERE node_from = $2 AND node_to = $3;"
|
||||
_, err = tx.Exec(sql, json_metadata, from, to)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deletes a relationship for the provided id
|
||||
func (conn *DB) DeleteRelation(from int64, to int64) error {
|
||||
tx, err := conn.Connection.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
|
||||
sql := "DELETE FROM relationships WHERE node_from = $1 AND node_to = $2;"
|
||||
conn.log(DEBUG, "Deleting relationship:", from, to)
|
||||
_, err = tx.Exec(sql, from, to)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in a new issue