package main import ( "context" "database/sql" "encoding/json" "fmt" "io" "net/http" "net/url" "strconv" "time" _ "modernc.org/sqlite" synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" ) const API_TOKEN = "" type ReadwiseCursor struct { Cursor string } type ReadwiseApiResponse[T, S any] struct { Results []T `json:"results"` Detail string `json:"detail"` Count uint64 `json:"count"` NextPageCursor S `json:"nextPageCursor"` } type RawReadwiseApiResponse struct { Results []json.RawMessage `json:"results"` // All ass raw } type ReadwiseDocument struct { Id string `json:"id"` Url string `json:"url"` Title string `json:"title"` // Author string `json:"author"` // Source string `json:"source"` // Category string `json:"category"` Location string `json:"location"` Tags map[string]ReadwiseTag `json:"tags"` // SiteName string `json:"site_name"` // CreatedAt string `json:"created_at"` // UpdatedAt string `json:"updated_at"` // Summary string `json:"summary"` SourceUrl string `json:"source_url"` // Notes string `json:"notes"` // ParentId interface{} `json:"parent_id"` // SavedAt string `json:"saved_at"` // LastMovedAt string `json:"last_moved_at"` } type ReadwiseTag struct { Name string `json:"name"` Type string `json:"type"` Created int `json:"created"` } type ReadwiseHighlight struct { UserBookID int `json:"user_book_id"` Title string `json:"title"` Author string `json:"author"` Source string `json:"source"` UniqueURL string `json:"unique_url"` BookTags []HighlightTag `json:"book_tags"` Category string `json:"category"` DocumentNote *string `json:"document_note"` ReadwiseURL string `json:"readwise_url"` SourceURL string `json:"source_url"` Highlights []HighlightItem `json:"highlights"` } type HighlightTag struct { Id int `json:"id"` Name string `json:"name"` } type HighlightItem struct { ID int `json:"id"` Text string `json:"text"` Location int `json:"location"` LocationType string `json:"location_type"` Note string `json:"note"` Color string `json:"color"` HighlightedAt string `json:"highlighted_at"` CreatedAt string `json:"created_at"` UpdatedAt string `json:"updated_at"` ExternalID string `json:"external_id"` EndLocation *int `json:"end_location"` URL string `json:"url"` BookID int `json:"book_id"` Tags []HighlightTag `json:"tags"` IsFavorite bool `json:"is_favorite"` IsDiscard bool `json:"is_discard"` ReadwiseURL string `json:"readwise_url"` } func sendReadwiseRequest[T, S any]( ctx context.Context, url string, ) (*ReadwiseApiResponse[T, S], *RawReadwiseApiResponse, error) { data := &ReadwiseApiResponse[T, S]{} raw := &RawReadwiseApiResponse{} req, err := http.NewRequest("GET", url, nil) if err != nil { fmt.Println("Error creating request:", err) return data, raw, err } // Add the authorization header req.Header.Set("Authorization", "Token "+API_TOKEN) client := &http.Client{} resp, err := client.Do(req) defer resp.Body.Close() if err != nil { return data, raw, err } body, err := io.ReadAll(resp.Body) if err != nil { return data, raw, err } err = json.Unmarshal(body, &data) if err != nil { return data, raw, err } if resp.StatusCode > 201 { return data, raw, fmt.Errorf( "Request failed with status %v: %v", resp.StatusCode, data.Detail, ) } err = json.Unmarshal(body, raw) if err != nil { return data, raw, err } return data, raw, nil } func getReadwiseDocuments( ctx context.Context, pagination synchronizator.Pagination, ) (synchronizator.FetchNodesResponse, error) { payload := synchronizator.FetchNodesResponse{ Pagination: pagination, } cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor) if !ok { return payload, fmt.Errorf("Couldn't retreive cursor from context!") } var documents []*synchronizator.Node params := url.Values{} params.Add("withHtmlContent", "true") if cursor.Cursor != "" { params.Add("pageCursor", cursor.Cursor) } url := "https://readwise.io/api/v3/list?" + params.Encode() data, rawData, err := sendReadwiseRequest[ReadwiseDocument, string](ctx, url) if err != nil { return payload, err } cursor.Cursor = data.NextPageCursor documents = make([]*synchronizator.Node, 0, len(data.Results)) for i, document := range data.Results { metadata, err := json.Marshal(document) if err != nil { return payload, err } node := synchronizator.NewNode( document.Title, "DOCUMENT", metadata, rawData.Results[i], ) documents = append(documents, node) } payload.Response = documents return payload, nil } func getReadwiseHighlights( ctx context.Context, pagination synchronizator.Pagination, ) (synchronizator.FetchNodesResponse, error) { payload := synchronizator.FetchNodesResponse{ Pagination: pagination, } cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor) if !ok { return payload, fmt.Errorf("Couldn't retreive cursor from context!") } var highlights []*synchronizator.Node params := url.Values{} if cursor.Cursor != "" { params.Add("pageCursor", cursor.Cursor) } url := "https://readwise.io/api/v2/export?" + params.Encode() data, rawData, err := sendReadwiseRequest[ReadwiseHighlight, int](ctx, url) if err != nil { return payload, err } if data.NextPageCursor != 0 { cursor.Cursor = strconv.Itoa(data.NextPageCursor) } else { cursor.Cursor = "" } highlights = make([]*synchronizator.Node, 0, len(data.Results)) for i, document := range data.Results { metadata, err := json.Marshal(document) if err != nil { return payload, err } node := synchronizator.NewNode( document.Title, "HIGHLIGHTS", metadata, rawData.Results[i], ) highlights = append(highlights, node) } payload.Response = highlights return payload, nil } func main() { start := time.Now() defer func() { elapsed := time.Now().Sub(start) fmt.Printf("\n\nExecution time took: %s", elapsed) }() connection, err := sql.Open("sqlite", "readwise.sql") if err != nil { fmt.Println(err) return } defer connection.Close() opts := synchronizator.DefaultOptions // opts.Log_level = synchronizator.DEBUG opts.DANGEROUSLY_DROP_TABLES = true sync, err := synchronizator.New(connection, opts) if err != nil { fmt.Println(err) return } readwise, err := sync.NewPlatform("READWISE", nil, nil) if err != nil { fmt.Println(err) return } pagination := synchronizator.StartPagination pagination.Pages = 1 pagination.Offset = 100 pagination.Total = 100 pagination.Limit = 100 pool_config := &synchronizator.WorkConfig{ AmountOfWorkers: 5, MaxRetries: 2, BaseRetryTime: time.Second * 30, RateLimit: synchronizator.NewRateLimiter(10, time.Minute), Timeout: time.Second * 2, } documents, err := readwise.AddCollection("DOCUMENTS", nil, nil) if err != nil { fmt.Println(err) return } cursor := &ReadwiseCursor{} ctx := context.WithValue(context.Background(), "readwise-cursor", cursor) for { err = documents.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config) if err != nil { fmt.Println(err) return } if cursor.Cursor == "" { break } } highlights, err := readwise.AddCollection("HIGHLIGHTS", nil, nil) if err != nil { fmt.Println(err) return } cursor.Cursor = "" for { err = highlights.FetchNodes(ctx, getReadwiseHighlights, pagination, pool_config) if err != nil { fmt.Println(err) return } if cursor.Cursor == "" { break } } }