synchronizator-go/examples/readwise/main.go

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"

	_ "modernc.org/sqlite"

	synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
)

const API_TOKEN = ""

type ReadwiseCursor struct {
	Cursor string
}

type ReadwiseApiResponse struct {
	Detail         string             `json:detail`
	Count          uint64             `json:"count"`
	NextPageCursor string             `json:"nextPageCursor"`
	Results        []ReadwiseDocument `json:"results"`
}

type RawReadwiseApiResponse struct {
	Count          uint64            `json:"count"`
	NextPageCursor string            `json:"nextPageCursor"`
	Results        []json.RawMessage `json:"results"` // All ass raw
}

type ReadwiseDocument struct {
	Id    string `json:"id"`
	Url   string `json:"url"`
	Title string `json:"title"`
	// Author      string            `json:"author"`
	// Source      string            `json:"source"`
	// Category    string            `json:"category"`
	Location string `json:"location"`
	// Tags        map[string]string `json:"tags"`
	// SiteName    string            `json:"site_name"`
	// CreatedAt   string            `json:"created_at"`
	// UpdatedAt   string            `json:"updated_at"`
	// Summary     string            `json:"summary"`
	SourceUrl string `json:"source_url"`
	// Notes       string            `json:"notes"`
	// ParentId    interface{}       `json:"parent_id"`
	// SavedAt     string            `json:"saved_at"`
	// LastMovedAt string            `json:"last_moved_at"`
}

func getReadwiseDocuments(
	ctx context.Context,
	pagination synchronizator.Pagination,
) (synchronizator.FetchNodesResponse, error) {
	payload := synchronizator.FetchNodesResponse{
		Pagination: pagination,
	}

	cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor)

	if !ok {
		return payload, fmt.Errorf("Couldn't retreive cursor from context!")
	}

	var documents []*synchronizator.Node

	params := url.Values{}
	params.Add("withHtmlContent", "true")
	if cursor.Cursor != "" {
		params.Add("pageCursor", cursor.Cursor)
	}

	url := "https://readwise.io/api/v3/list?" + params.Encode()
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println("Error creating request:", err)
		return payload, err
	}

	// Add the authorization header
	req.Header.Set("Authorization", "Token "+API_TOKEN)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return payload, err
	}

	body, err := io.ReadAll(resp.Body)
	resp.Body.Close()

	var data ReadwiseApiResponse
	err = json.Unmarshal(body, &data)
	if err != nil {
		return payload, err
	}

	if resp.StatusCode > 201 {
		return payload, fmt.Errorf(
			"Request failed with status %v: %v",
			resp.StatusCode,
			data.Detail,
		)
	}

	var rawData RawReadwiseApiResponse
	err = json.Unmarshal(body, &rawData)
	if err != nil {
		return payload, err
	}

	cursor.Cursor = data.NextPageCursor

	documents = make([]*synchronizator.Node, 0, len(data.Results))

	for i, document := range data.Results {
		metadata, err := json.Marshal(document)
		if err != nil {
			return payload, err
		}

		node := synchronizator.NewNode(
			document.Title,
			"DOCUMENT",
			metadata,
			rawData.Results[i],
		)
		documents = append(documents, node)
	}

	payload.Response = documents

	return payload, nil
}

func main() {
	start := time.Now()

	defer func() {
		elapsed := time.Now().Sub(start)
		fmt.Printf("\n\nExecution time took: %s", elapsed)
	}()

	connection, err := sql.Open("sqlite", "readwise.sql")
	if err != nil {
		fmt.Println(err)

		return
	}

	defer connection.Close()

	opts := synchronizator.DefaultOptions
	// opts.Log_level = synchronizator.DEBUG
	opts.DANGEROUSLY_DROP_TABLES = true

	sync, err := synchronizator.New(connection, opts)
	if err != nil {
		fmt.Println(err)

		return
	}

	readwiseReader, err := sync.NewPlatform("readwise_reader", nil, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	pagination := synchronizator.StartPagination
	pagination.Pages = 1
	pagination.Offset = 100
	pagination.Total = 100
	pagination.Limit = 100
	pool_config := &synchronizator.WorkConfig{
		AmountOfWorkers: 5,
		MaxRetries:      2,
		BaseRetryTime:   time.Second * 30,
		RateLimit:       synchronizator.NewRateLimiter(10, time.Minute),
		Timeout:         time.Second * 2,
	}

	collection, err := readwiseReader.GetDefaultCollection()

	cursor := &ReadwiseCursor{}

	ctx := context.WithValue(context.Background(), "readwise-cursor", cursor)

	for {
		err = collection.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
		if err != nil {
			fmt.Println(err)
			return
		}

		if cursor.Cursor == "" {
			break
		}
	}
}