synchronizator-go/examples/readwise/main.go

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"time"

	_ "modernc.org/sqlite"

	synchronizator "git.alecodes.page/alecodes/synchronizator/pkg"
)

const API_TOKEN = ""

type ReadwiseCursor struct {
	Cursor string
}

type ReadwiseApiResponse[T, S any] struct {
	Results        []T    `json:"results"`
	Detail         string `json:detail`
	Count          uint64 `json:"count"`
	NextPageCursor S      `json:"nextPageCursor"`
}

type RawReadwiseApiResponse struct {
	Results []json.RawMessage `json:"results"` // All ass raw
}

type ReadwiseDocument struct {
	Id    string `json:"id"`
	Url   string `json:"url"`
	Title string `json:"title"`
	// Author      string            `json:"author"`
	// Source      string            `json:"source"`
	// Category    string            `json:"category"`
	Location string `json:"location"`
	// Tags        map[string]string `json:"tags"`
	// SiteName    string            `json:"site_name"`
	// CreatedAt   string            `json:"created_at"`
	// UpdatedAt   string            `json:"updated_at"`
	// Summary     string            `json:"summary"`
	SourceUrl string `json:"source_url"`
	// Notes       string            `json:"notes"`
	// ParentId    interface{}       `json:"parent_id"`
	// SavedAt     string            `json:"saved_at"`
	// LastMovedAt string            `json:"last_moved_at"`
}

type ReadwiseHighlight struct {
	UserBookID   int             `json:"user_book_id"`
	Title        string          `json:"title"`
	Author       string          `json:"author"`
	Source       string          `json:"source"`
	UniqueURL    string          `json:"unique_url"`
	BookTags     []HighlightTag  `json:"book_tags"`
	Category     string          `json:"category"`
	DocumentNote *string         `json:"document_note"`
	ReadwiseURL  string          `json:"readwise_url"`
	SourceURL    string          `json:"source_url"`
	Highlights   []HighlightItem `json:"highlights"`
}

type HighlightTag struct {
	Id   int    `json:id`
	Name string `json:name`
}

type HighlightItem struct {
	ID            int            `json:"id"`
	Text          string         `json:"text"`
	Location      int            `json:"location"`
	LocationType  string         `json:"location_type"`
	Note          string         `json:"note"`
	Color         string         `json:"color"`
	HighlightedAt string         `json:"highlighted_at"`
	CreatedAt     string         `json:"created_at"`
	UpdatedAt     string         `json:"updated_at"`
	ExternalID    string         `json:"external_id"`
	EndLocation   *int           `json:"end_location"`
	URL           string         `json:"url"`
	BookID        int            `json:"book_id"`
	Tags          []HighlightTag `json:"tags"`
	IsFavorite    bool           `json:"is_favorite"`
	IsDiscard     bool           `json:"is_discard"`
	ReadwiseURL   string         `json:"readwise_url"`
}

func sendReadwiseRequest[T, S any](
	ctx context.Context,
	url string,
) (*ReadwiseApiResponse[T, S], *RawReadwiseApiResponse, error) {
	data := &ReadwiseApiResponse[T, S]{}
	raw := &RawReadwiseApiResponse{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println("Error creating request:", err)
		return data, raw, err
	}

	// Add the authorization header
	req.Header.Set("Authorization", "Token "+API_TOKEN)

	client := &http.Client{}
	resp, err := client.Do(req)
	defer resp.Body.Close()
	if err != nil {
		return data, raw, err
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return data, raw, err
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return data, raw, err
	}

	if resp.StatusCode > 201 {
		return data, raw, fmt.Errorf(
			"Request failed with status %v: %v",
			resp.StatusCode,
			data.Detail,
		)
	}

	err = json.Unmarshal(body, raw)
	if err != nil {
		return data, raw, err
	}

	return data, raw, nil
}

func getReadwiseDocuments(
	ctx context.Context,
	pagination synchronizator.Pagination,
) (synchronizator.FetchNodesResponse, error) {
	payload := synchronizator.FetchNodesResponse{
		Pagination: pagination,
	}

	cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor)

	if !ok {
		return payload, fmt.Errorf("Couldn't retreive cursor from context!")
	}

	var documents []*synchronizator.Node

	params := url.Values{}
	params.Add("withHtmlContent", "true")
	if cursor.Cursor != "" {
		params.Add("pageCursor", cursor.Cursor)
	}

	url := "https://readwise.io/api/v3/list?" + params.Encode()
	data, rawData, err := sendReadwiseRequest[ReadwiseDocument, string](ctx, url)
	if err != nil {
		return payload, err
	}

	cursor.Cursor = data.NextPageCursor

	documents = make([]*synchronizator.Node, 0, len(data.Results))

	for i, document := range data.Results {
		metadata, err := json.Marshal(document)
		if err != nil {
			return payload, err
		}

		node := synchronizator.NewNode(
			document.Title,
			"DOCUMENT",
			metadata,
			rawData.Results[i],
		)
		documents = append(documents, node)
	}

	payload.Response = documents

	return payload, nil
}

func getReadwiseHighlights(
	ctx context.Context,
	pagination synchronizator.Pagination,
) (synchronizator.FetchNodesResponse, error) {
	payload := synchronizator.FetchNodesResponse{
		Pagination: pagination,
	}

	cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor)

	if !ok {
		return payload, fmt.Errorf("Couldn't retreive cursor from context!")
	}

	var highlights []*synchronizator.Node

	params := url.Values{}
	if cursor.Cursor != "" {
		params.Add("pageCursor", cursor.Cursor)
	}

	url := "https://readwise.io/api/v2/export?" + params.Encode()
	data, rawData, err := sendReadwiseRequest[ReadwiseHighlight, int](ctx, url)
	if err != nil {
		return payload, err
	}

	if data.NextPageCursor != 0 {
		cursor.Cursor = strconv.Itoa(data.NextPageCursor)
	} else {
		cursor.Cursor = ""
	}

	highlights = make([]*synchronizator.Node, 0, len(data.Results))

	for i, document := range data.Results {
		metadata, err := json.Marshal(document)
		if err != nil {
			return payload, err
		}

		node := synchronizator.NewNode(
			document.Title,
			"HIGHLIGHTS",
			metadata,
			rawData.Results[i],
		)
		highlights = append(highlights, node)
	}

	payload.Response = highlights

	return payload, nil
}

func main() {
	start := time.Now()

	defer func() {
		elapsed := time.Now().Sub(start)
		fmt.Printf("\n\nExecution time took: %s", elapsed)
	}()

	connection, err := sql.Open("sqlite", "readwise.sql")
	if err != nil {
		fmt.Println(err)

		return
	}

	defer connection.Close()

	opts := synchronizator.DefaultOptions
	// opts.Log_level = synchronizator.DEBUG
	opts.DANGEROUSLY_DROP_TABLES = true

	sync, err := synchronizator.New(connection, opts)
	if err != nil {
		fmt.Println(err)

		return
	}

	readwise, err := sync.NewPlatform("READWISE", nil, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	pagination := synchronizator.StartPagination
	pagination.Pages = 1
	pagination.Offset = 100
	pagination.Total = 100
	pagination.Limit = 100
	pool_config := &synchronizator.WorkConfig{
		AmountOfWorkers: 5,
		MaxRetries:      2,
		BaseRetryTime:   time.Second * 30,
		RateLimit:       synchronizator.NewRateLimiter(10, time.Minute),
		Timeout:         time.Second * 2,
	}

	documents, err := readwise.AddCollection("DOCUMENTS", nil, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	cursor := &ReadwiseCursor{}

	ctx := context.WithValue(context.Background(), "readwise-cursor", cursor)

	for {
		err = documents.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
		if err != nil {
			fmt.Println(err)
			return
		}

		if cursor.Cursor == "" {
			break
		}
	}

	highlights, err := readwise.AddCollection("HIGHLIGHTS", nil, nil)
	if err != nil {
		fmt.Println(err)
		return
	}

	cursor.Cursor = ""
	for {
		err = highlights.FetchNodes(ctx, getReadwiseHighlights, pagination, pool_config)
		if err != nil {
			fmt.Println(err)
			return
		}

		if cursor.Cursor == "" {
			break
		}
	}
}