feat: add basic readwise example

examples/readwise/main.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	_ "modernc.org/sqlite"
+
+	synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
+)
+
+const API_TOKEN = ""
+
+type ReadwiseCursor struct {
+	Cursor string
+}
+
+type ReadwiseApiResponse struct {
+	Count          uint64             `json:"count"`
+	NextPageCursor string             `json:"nextPageCursor"`
+	Results        []ReadwiseDocument `json:"results"`
+}
+
+type RawReadwiseApiResponse struct {
+	Count          uint64            `json:"count"`
+	NextPageCursor string            `json:"nextPageCursor"`
+	Results        []json.RawMessage `json:"results"` // All ass raw
+}
+
+type ReadwiseDocument struct {
+	Id    string `json:"id"`
+	Url   string `json:"url"`
+	Title string `json:"title"`
+	// Author      string            `json:"author"`
+	// Source      string            `json:"source"`
+	// Category    string            `json:"category"`
+	Location string `json:"location"`
+	// Tags        map[string]string `json:"tags"`
+	// SiteName    string            `json:"site_name"`
+	// CreatedAt   string            `json:"created_at"`
+	// UpdatedAt   string            `json:"updated_at"`
+	// Summary     string            `json:"summary"`
+	SourceUrl string `json:"source_url"`
+	// Notes       string            `json:"notes"`
+	// ParentId    interface{}       `json:"parent_id"`
+	// SavedAt     string            `json:"saved_at"`
+	// LastMovedAt string            `json:"last_moved_at"`
+}
+
+func getReadwiseDocuments(
+	ctx context.Context,
+	pagination synchronizator.Pagination,
+) (synchronizator.FetchNodesResponse, error) {
+	payload := synchronizator.FetchNodesResponse{
+		Pagination: pagination,
+	}
+
+	cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor)
+
+	if !ok {
+		return payload, fmt.Errorf("Couldn't retreive cursor from context!")
+	}
+
+	var documents []*synchronizator.Node
+
+	params := url.Values{}
+	if cursor.Cursor != "" {
+		params.Add("pageCursor", cursor.Cursor)
+	}
+
+	url := "https://readwise.io/api/v3/list?" + params.Encode()
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		fmt.Println("Error creating request:", err)
+		return payload, err
+	}
+
+	// Add the authorization header
+	req.Header.Set("Authorization", "Token "+API_TOKEN)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return payload, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	resp.Body.Close()
+
+	var data ReadwiseApiResponse
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return payload, err
+	}
+
+	var rawData RawReadwiseApiResponse
+	err = json.Unmarshal(body, &rawData)
+	if err != nil {
+		return payload, err
+	}
+
+	cursor.Cursor = data.NextPageCursor
+
+	documents = make([]*synchronizator.Node, 0, len(data.Results))
+
+	for i, document := range data.Results {
+		metadata, err := json.Marshal(document)
+		if err != nil {
+			return payload, err
+		}
+
+		node := synchronizator.NewNode(
+			document.Title,
+			"DOCUMENT",
+			metadata,
+			rawData.Results[i],
+		)
+		documents = append(documents, node)
+	}
+
+	payload.Response = documents
+
+	return payload, nil
+}
+
+func main() {
+	start := time.Now()
+
+	defer func() {
+		elapsed := time.Now().Sub(start)
+		fmt.Printf("\n\nExecution time took: %s", elapsed)
+	}()
+
+	connection, err := sql.Open("sqlite", "readwise.sql")
+	if err != nil {
+		fmt.Println(err)
+
+		return
+	}
+
+	defer connection.Close()
+
+	opts := synchronizator.DefaultOptions
+	// opts.Log_level = synchronizator.DEBUG
+	opts.DANGEROUSLY_DROP_TABLES = true
+
+	sync, err := synchronizator.New(connection, opts)
+	if err != nil {
+		fmt.Println(err)
+
+		return
+	}
+
+	readwiseReader, err := sync.NewPlatform("readwise_reader", nil, nil)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	pagination := synchronizator.StartPagination
+	pagination.Pages = 0
+	pagination.Total = 100
+	pagination.Limit = 100
+	pool_config := &synchronizator.WorkConfig{
+		AmountOfWorkers: 5,
+		MaxRetries:      1,
+		BaseRetryTime:   time.Second * 2,
+		RateLimit:       synchronizator.NewRateLimiter(20, time.Minute),
+		Timeout:         time.Second * 2,
+	}
+
+	collection, err := readwiseReader.GetDefaultCollection()
+
+	cursor := &ReadwiseCursor{}
+
+	ctx := context.WithValue(context.Background(), "readwise-cursor", cursor)
+
+	for {
+		err = collection.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+		if cursor.Cursor == "" {
+			break
+		}
+	}
+}