diff --git a/examples/readwise/main.go b/examples/readwise/main.go index b85c660..284f0c2 100644 --- a/examples/readwise/main.go +++ b/examples/readwise/main.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strconv" "time" _ "modernc.org/sqlite" @@ -21,17 +22,15 @@ type ReadwiseCursor struct { Cursor string } -type ReadwiseApiResponse struct { - Detail string `json:detail` - Count uint64 `json:"count"` - NextPageCursor string `json:"nextPageCursor"` - Results []ReadwiseDocument `json:"results"` +type ReadwiseApiResponse[T, S any] struct { + Results []T `json:"results"` + Detail string `json:detail` + Count uint64 `json:"count"` + NextPageCursor S `json:"nextPageCursor"` } type RawReadwiseApiResponse struct { - Count uint64 `json:"count"` - NextPageCursor string `json:"nextPageCursor"` - Results []json.RawMessage `json:"results"` // All ass raw + Results []json.RawMessage `json:"results"` // All ass raw } type ReadwiseDocument struct { @@ -54,6 +53,94 @@ type ReadwiseDocument struct { // LastMovedAt string `json:"last_moved_at"` } +type ReadwiseHighlight struct { + UserBookID int `json:"user_book_id"` + Title string `json:"title"` + Author string `json:"author"` + Source string `json:"source"` + UniqueURL string `json:"unique_url"` + BookTags []HighlightTag `json:"book_tags"` + Category string `json:"category"` + DocumentNote *string `json:"document_note"` + ReadwiseURL string `json:"readwise_url"` + SourceURL string `json:"source_url"` + Highlights []HighlightItem `json:"highlights"` +} + +type HighlightTag struct { + Id int `json:id` + Name string `json:name` +} + +type HighlightItem struct { + ID int `json:"id"` + Text string `json:"text"` + Location int `json:"location"` + LocationType string `json:"location_type"` + Note string `json:"note"` + Color string `json:"color"` + HighlightedAt string `json:"highlighted_at"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + ExternalID string `json:"external_id"` + EndLocation *int `json:"end_location"` + URL string `json:"url"` + BookID int `json:"book_id"` + Tags []HighlightTag `json:"tags"` + IsFavorite bool `json:"is_favorite"` + IsDiscard bool `json:"is_discard"` + ReadwiseURL string `json:"readwise_url"` +} + +func sendReadwiseRequest[T, S any]( + ctx context.Context, + url string, +) (*ReadwiseApiResponse[T, S], *RawReadwiseApiResponse, error) { + data := &ReadwiseApiResponse[T, S]{} + raw := &RawReadwiseApiResponse{} + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + fmt.Println("Error creating request:", err) + return data, raw, err + } + + // Add the authorization header + req.Header.Set("Authorization", "Token "+API_TOKEN) + + client := &http.Client{} + resp, err := client.Do(req) + defer resp.Body.Close() + if err != nil { + return data, raw, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return data, raw, err + } + + err = json.Unmarshal(body, &data) + if err != nil { + return data, raw, err + } + + if resp.StatusCode > 201 { + return data, raw, fmt.Errorf( + "Request failed with status %v: %v", + resp.StatusCode, + data.Detail, + ) + } + + err = json.Unmarshal(body, raw) + if err != nil { + return data, raw, err + } + + return data, raw, nil +} + func getReadwiseDocuments( ctx context.Context, pagination synchronizator.Pagination, @@ -77,40 +164,7 @@ func getReadwiseDocuments( } url := "https://readwise.io/api/v3/list?" + params.Encode() - req, err := http.NewRequest("GET", url, nil) - if err != nil { - fmt.Println("Error creating request:", err) - return payload, err - } - - // Add the authorization header - req.Header.Set("Authorization", "Token "+API_TOKEN) - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return payload, err - } - - body, err := io.ReadAll(resp.Body) - resp.Body.Close() - - var data ReadwiseApiResponse - err = json.Unmarshal(body, &data) - if err != nil { - return payload, err - } - - if resp.StatusCode > 201 { - return payload, fmt.Errorf( - "Request failed with status %v: %v", - resp.StatusCode, - data.Detail, - ) - } - - var rawData RawReadwiseApiResponse - err = json.Unmarshal(body, &rawData) + data, rawData, err := sendReadwiseRequest[ReadwiseDocument, string](ctx, url) if err != nil { return payload, err } @@ -139,6 +193,61 @@ func getReadwiseDocuments( return payload, nil } +func getReadwiseHighlights( + ctx context.Context, + pagination synchronizator.Pagination, +) (synchronizator.FetchNodesResponse, error) { + payload := synchronizator.FetchNodesResponse{ + Pagination: pagination, + } + + cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor) + + if !ok { + return payload, fmt.Errorf("Couldn't retreive cursor from context!") + } + + var highlights []*synchronizator.Node + + params := url.Values{} + if cursor.Cursor != "" { + params.Add("pageCursor", cursor.Cursor) + } + + url := "https://readwise.io/api/v2/export?" + params.Encode() + data, rawData, err := sendReadwiseRequest[ReadwiseHighlight, int](ctx, url) + if err != nil { + return payload, err + } + + if data.NextPageCursor != 0 { + cursor.Cursor = strconv.Itoa(data.NextPageCursor) + } else { + cursor.Cursor = "" + } + + highlights = make([]*synchronizator.Node, 0, len(data.Results)) + + for i, document := range data.Results { + metadata, err := json.Marshal(document) + if err != nil { + return payload, err + } + + node := synchronizator.NewNode( + document.Title, + "HIGHLIGHTS", + metadata, + rawData.Results[i], + ) + highlights = append(highlights, node) + } + + payload.Response = highlights + + return payload, nil +} + func main() { start := time.Now() @@ -167,7 +276,7 @@ func main() { return } - readwiseReader, err := sync.NewPlatform("readwise_reader", nil, nil) + readwise, err := sync.NewPlatform("READWISE", nil, nil) if err != nil { fmt.Println(err) return @@ -186,14 +295,37 @@ func main() { Timeout: time.Second * 2, } - collection, err := readwiseReader.GetDefaultCollection() + documents, err := readwise.AddCollection("DOCUMENTS", nil, nil) + if err != nil { + fmt.Println(err) + return + } cursor := &ReadwiseCursor{} ctx := context.WithValue(context.Background(), "readwise-cursor", cursor) for { - err = collection.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config) + err = documents.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config) + if err != nil { + fmt.Println(err) + return + } + + if cursor.Cursor == "" { + break + } + } + + highlights, err := readwise.AddCollection("HIGHLIGHTS", nil, nil) + if err != nil { + fmt.Println(err) + return + } + + cursor.Cursor = "" + for { + err = highlights.FetchNodes(ctx, getReadwiseHighlights, pagination, pool_config) if err != nil { fmt.Println(err) return diff --git a/pkg/collection.go b/pkg/collection.go index bcb4678..a7592e5 100644 --- a/pkg/collection.go +++ b/pkg/collection.go @@ -107,6 +107,7 @@ func (collection *Collection) FetchNodes( startPagination Pagination, poolConfig *WorkConfig, ) error { + fmt.Printf("Fetching nodes for Collection: %d - %v\n", collection.Id, collection.name) values, err := fetchWithPagination(ctx, poolConfig, fetcher, startPagination) if err != nil { return err diff --git a/pkg/platform.go b/pkg/platform.go index 7c84d4f..9348343 100644 --- a/pkg/platform.go +++ b/pkg/platform.go @@ -13,6 +13,20 @@ type Platform struct { Collections []*Collection // Child nodes } +func (platform *Platform) AddCollection( + name string, + metadata, originalData []byte, +) (*Collection, error) { + collection, err := platform._conn.NewCollection(name, metadata, originalData) + if err != nil { + return nil, err + } + + platform.Collections = append(platform.Collections, collection) + + return collection, nil +} + func (platform *Platform) GetDefaultCollection() (*Collection, error) { for _, collection := range platform.Collections { if collection.IsDefault() {