feat: expand readwise example

examples/readwise/main.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
 	_ "modernc.org/sqlite"
@@ -21,16 +22,14 @@ type ReadwiseCursor struct {
 	Cursor string
 }
 
-type ReadwiseApiResponse struct {
+type ReadwiseApiResponse[T, S any] struct {
+	Results        []T    `json:"results"`
 	Detail         string `json:detail`
 	Count          uint64 `json:"count"`
-	NextPageCursor string             `json:"nextPageCursor"`
+	NextPageCursor S      `json:"nextPageCursor"`
-	Results        []ReadwiseDocument `json:"results"`
 }
 
 type RawReadwiseApiResponse struct {
-	Count          uint64            `json:"count"`
-	NextPageCursor string            `json:"nextPageCursor"`
 	Results []json.RawMessage `json:"results"` // All ass raw
 }
 
@@ -54,6 +53,94 @@ type ReadwiseDocument struct {
 	// LastMovedAt string            `json:"last_moved_at"`
 }
 
+type ReadwiseHighlight struct {
+	UserBookID   int             `json:"user_book_id"`
+	Title        string          `json:"title"`
+	Author       string          `json:"author"`
+	Source       string          `json:"source"`
+	UniqueURL    string          `json:"unique_url"`
+	BookTags     []HighlightTag  `json:"book_tags"`
+	Category     string          `json:"category"`
+	DocumentNote *string         `json:"document_note"`
+	ReadwiseURL  string          `json:"readwise_url"`
+	SourceURL    string          `json:"source_url"`
+	Highlights   []HighlightItem `json:"highlights"`
+}
+
+type HighlightTag struct {
+	Id   int    `json:id`
+	Name string `json:name`
+}
+
+type HighlightItem struct {
+	ID            int            `json:"id"`
+	Text          string         `json:"text"`
+	Location      int            `json:"location"`
+	LocationType  string         `json:"location_type"`
+	Note          string         `json:"note"`
+	Color         string         `json:"color"`
+	HighlightedAt string         `json:"highlighted_at"`
+	CreatedAt     string         `json:"created_at"`
+	UpdatedAt     string         `json:"updated_at"`
+	ExternalID    string         `json:"external_id"`
+	EndLocation   *int           `json:"end_location"`
+	URL           string         `json:"url"`
+	BookID        int            `json:"book_id"`
+	Tags          []HighlightTag `json:"tags"`
+	IsFavorite    bool           `json:"is_favorite"`
+	IsDiscard     bool           `json:"is_discard"`
+	ReadwiseURL   string         `json:"readwise_url"`
+}
+
+func sendReadwiseRequest[T, S any](
+	ctx context.Context,
+	url string,
+) (*ReadwiseApiResponse[T, S], *RawReadwiseApiResponse, error) {
+	data := &ReadwiseApiResponse[T, S]{}
+	raw := &RawReadwiseApiResponse{}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		fmt.Println("Error creating request:", err)
+		return data, raw, err
+	}
+
+	// Add the authorization header
+	req.Header.Set("Authorization", "Token "+API_TOKEN)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	defer resp.Body.Close()
+	if err != nil {
+		return data, raw, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return data, raw, err
+	}
+
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return data, raw, err
+	}
+
+	if resp.StatusCode > 201 {
+		return data, raw, fmt.Errorf(
+			"Request failed with status %v: %v",
+			resp.StatusCode,
+			data.Detail,
+		)
+	}
+
+	err = json.Unmarshal(body, raw)
+	if err != nil {
+		return data, raw, err
+	}
+
+	return data, raw, nil
+}
+
 func getReadwiseDocuments(
 	ctx context.Context,
 	pagination synchronizator.Pagination,
@@ -77,40 +164,7 @@ func getReadwiseDocuments(
 	}
 
 	url := "https://readwise.io/api/v3/list?" + params.Encode()
-	req, err := http.NewRequest("GET", url, nil)
+	data, rawData, err := sendReadwiseRequest[ReadwiseDocument, string](ctx, url)
-	if err != nil {
-		fmt.Println("Error creating request:", err)
-		return payload, err
-	}
-
-	// Add the authorization header
-	req.Header.Set("Authorization", "Token "+API_TOKEN)
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return payload, err
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	resp.Body.Close()
-
-	var data ReadwiseApiResponse
-	err = json.Unmarshal(body, &data)
-	if err != nil {
-		return payload, err
-	}
-
-	if resp.StatusCode > 201 {
-		return payload, fmt.Errorf(
-			"Request failed with status %v: %v",
-			resp.StatusCode,
-			data.Detail,
-		)
-	}
-
-	var rawData RawReadwiseApiResponse
-	err = json.Unmarshal(body, &rawData)
 	if err != nil {
 		return payload, err
 	}
@@ -139,6 +193,61 @@ func getReadwiseDocuments(
 	return payload, nil
 }
 
+func getReadwiseHighlights(
+	ctx context.Context,
+	pagination synchronizator.Pagination,
+) (synchronizator.FetchNodesResponse, error) {
+	payload := synchronizator.FetchNodesResponse{
+		Pagination: pagination,
+	}
+
+	cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor)
+
+	if !ok {
+		return payload, fmt.Errorf("Couldn't retreive cursor from context!")
+	}
+
+	var highlights []*synchronizator.Node
+
+	params := url.Values{}
+	if cursor.Cursor != "" {
+		params.Add("pageCursor", cursor.Cursor)
+	}
+
+	url := "https://readwise.io/api/v2/export?" + params.Encode()
+	data, rawData, err := sendReadwiseRequest[ReadwiseHighlight, int](ctx, url)
+	if err != nil {
+		return payload, err
+	}
+
+	if data.NextPageCursor != 0 {
+		cursor.Cursor = strconv.Itoa(data.NextPageCursor)
+	} else {
+		cursor.Cursor = ""
+	}
+
+	highlights = make([]*synchronizator.Node, 0, len(data.Results))
+
+	for i, document := range data.Results {
+		metadata, err := json.Marshal(document)
+		if err != nil {
+			return payload, err
+		}
+
+		node := synchronizator.NewNode(
+			document.Title,
+			"HIGHLIGHTS",
+			metadata,
+			rawData.Results[i],
+		)
+		highlights = append(highlights, node)
+	}
+
+	payload.Response = highlights
+
+	return payload, nil
+}
+
 func main() {
 	start := time.Now()
 
@@ -167,7 +276,7 @@ func main() {
 		return
 	}
 
-	readwiseReader, err := sync.NewPlatform("readwise_reader", nil, nil)
+	readwise, err := sync.NewPlatform("READWISE", nil, nil)
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -186,14 +295,37 @@ func main() {
 		Timeout:         time.Second * 2,
 	}
 
-	collection, err := readwiseReader.GetDefaultCollection()
+	documents, err := readwise.AddCollection("DOCUMENTS", nil, nil)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
 
 	cursor := &ReadwiseCursor{}
 
 	ctx := context.WithValue(context.Background(), "readwise-cursor", cursor)
 
 	for {
-		err = collection.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
+		err = documents.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+		if cursor.Cursor == "" {
+			break
+		}
+	}
+
+	highlights, err := readwise.AddCollection("HIGHLIGHTS", nil, nil)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	cursor.Cursor = ""
+	for {
+		err = highlights.FetchNodes(ctx, getReadwiseHighlights, pagination, pool_config)
 		if err != nil {
 			fmt.Println(err)
 			return

pkg/collection.go

@@ -107,6 +107,7 @@ func (collection *Collection) FetchNodes(
 	startPagination Pagination,
 	poolConfig *WorkConfig,
 ) error {
+	fmt.Printf("Fetching nodes for Collection: %d - %v\n", collection.Id, collection.name)
 	values, err := fetchWithPagination(ctx, poolConfig, fetcher, startPagination)
 	if err != nil {
 		return err

pkg/platform.go

@@ -13,6 +13,20 @@ type Platform struct {
 	Collections []*Collection // Child nodes
 }
 
+func (platform *Platform) AddCollection(
+	name string,
+	metadata, originalData []byte,
+) (*Collection, error) {
+	collection, err := platform._conn.NewCollection(name, metadata, originalData)
+	if err != nil {
+		return nil, err
+	}
+
+	platform.Collections = append(platform.Collections, collection)
+
+	return collection, nil
+}
+
 func (platform *Platform) GetDefaultCollection() (*Collection, error) {
 	for _, collection := range platform.Collections {
 		if collection.IsDefault() {