generated from alecodes/base-template
feat: expand readwise example
This commit is contained in:
parent
8f424d45f7
commit
5d00b7c336
3 changed files with 192 additions and 45 deletions
|
|
@ -8,6 +8,7 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
|
@ -21,17 +22,15 @@ type ReadwiseCursor struct {
|
|||
Cursor string
|
||||
}
|
||||
|
||||
type ReadwiseApiResponse struct {
|
||||
Detail string `json:detail`
|
||||
Count uint64 `json:"count"`
|
||||
NextPageCursor string `json:"nextPageCursor"`
|
||||
Results []ReadwiseDocument `json:"results"`
|
||||
type ReadwiseApiResponse[T, S any] struct {
|
||||
Results []T `json:"results"`
|
||||
Detail string `json:detail`
|
||||
Count uint64 `json:"count"`
|
||||
NextPageCursor S `json:"nextPageCursor"`
|
||||
}
|
||||
|
||||
type RawReadwiseApiResponse struct {
|
||||
Count uint64 `json:"count"`
|
||||
NextPageCursor string `json:"nextPageCursor"`
|
||||
Results []json.RawMessage `json:"results"` // All ass raw
|
||||
Results []json.RawMessage `json:"results"` // All ass raw
|
||||
}
|
||||
|
||||
type ReadwiseDocument struct {
|
||||
|
|
@ -54,6 +53,94 @@ type ReadwiseDocument struct {
|
|||
// LastMovedAt string `json:"last_moved_at"`
|
||||
}
|
||||
|
||||
type ReadwiseHighlight struct {
|
||||
UserBookID int `json:"user_book_id"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Source string `json:"source"`
|
||||
UniqueURL string `json:"unique_url"`
|
||||
BookTags []HighlightTag `json:"book_tags"`
|
||||
Category string `json:"category"`
|
||||
DocumentNote *string `json:"document_note"`
|
||||
ReadwiseURL string `json:"readwise_url"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Highlights []HighlightItem `json:"highlights"`
|
||||
}
|
||||
|
||||
type HighlightTag struct {
|
||||
Id int `json:id`
|
||||
Name string `json:name`
|
||||
}
|
||||
|
||||
type HighlightItem struct {
|
||||
ID int `json:"id"`
|
||||
Text string `json:"text"`
|
||||
Location int `json:"location"`
|
||||
LocationType string `json:"location_type"`
|
||||
Note string `json:"note"`
|
||||
Color string `json:"color"`
|
||||
HighlightedAt string `json:"highlighted_at"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
ExternalID string `json:"external_id"`
|
||||
EndLocation *int `json:"end_location"`
|
||||
URL string `json:"url"`
|
||||
BookID int `json:"book_id"`
|
||||
Tags []HighlightTag `json:"tags"`
|
||||
IsFavorite bool `json:"is_favorite"`
|
||||
IsDiscard bool `json:"is_discard"`
|
||||
ReadwiseURL string `json:"readwise_url"`
|
||||
}
|
||||
|
||||
func sendReadwiseRequest[T, S any](
|
||||
ctx context.Context,
|
||||
url string,
|
||||
) (*ReadwiseApiResponse[T, S], *RawReadwiseApiResponse, error) {
|
||||
data := &ReadwiseApiResponse[T, S]{}
|
||||
raw := &RawReadwiseApiResponse{}
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
fmt.Println("Error creating request:", err)
|
||||
return data, raw, err
|
||||
}
|
||||
|
||||
// Add the authorization header
|
||||
req.Header.Set("Authorization", "Token "+API_TOKEN)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
defer resp.Body.Close()
|
||||
if err != nil {
|
||||
return data, raw, err
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return data, raw, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &data)
|
||||
if err != nil {
|
||||
return data, raw, err
|
||||
}
|
||||
|
||||
if resp.StatusCode > 201 {
|
||||
return data, raw, fmt.Errorf(
|
||||
"Request failed with status %v: %v",
|
||||
resp.StatusCode,
|
||||
data.Detail,
|
||||
)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, raw)
|
||||
if err != nil {
|
||||
return data, raw, err
|
||||
}
|
||||
|
||||
return data, raw, nil
|
||||
}
|
||||
|
||||
func getReadwiseDocuments(
|
||||
ctx context.Context,
|
||||
pagination synchronizator.Pagination,
|
||||
|
|
@ -77,40 +164,7 @@ func getReadwiseDocuments(
|
|||
}
|
||||
|
||||
url := "https://readwise.io/api/v3/list?" + params.Encode()
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
fmt.Println("Error creating request:", err)
|
||||
return payload, err
|
||||
}
|
||||
|
||||
// Add the authorization header
|
||||
req.Header.Set("Authorization", "Token "+API_TOKEN)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
var data ReadwiseApiResponse
|
||||
err = json.Unmarshal(body, &data)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
|
||||
if resp.StatusCode > 201 {
|
||||
return payload, fmt.Errorf(
|
||||
"Request failed with status %v: %v",
|
||||
resp.StatusCode,
|
||||
data.Detail,
|
||||
)
|
||||
}
|
||||
|
||||
var rawData RawReadwiseApiResponse
|
||||
err = json.Unmarshal(body, &rawData)
|
||||
data, rawData, err := sendReadwiseRequest[ReadwiseDocument, string](ctx, url)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
|
|
@ -139,6 +193,61 @@ func getReadwiseDocuments(
|
|||
return payload, nil
|
||||
}
|
||||
|
||||
func getReadwiseHighlights(
|
||||
ctx context.Context,
|
||||
pagination synchronizator.Pagination,
|
||||
) (synchronizator.FetchNodesResponse, error) {
|
||||
payload := synchronizator.FetchNodesResponse{
|
||||
Pagination: pagination,
|
||||
}
|
||||
|
||||
cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor)
|
||||
|
||||
if !ok {
|
||||
return payload, fmt.Errorf("Couldn't retreive cursor from context!")
|
||||
}
|
||||
|
||||
var highlights []*synchronizator.Node
|
||||
|
||||
params := url.Values{}
|
||||
if cursor.Cursor != "" {
|
||||
params.Add("pageCursor", cursor.Cursor)
|
||||
}
|
||||
|
||||
url := "https://readwise.io/api/v2/export?" + params.Encode()
|
||||
data, rawData, err := sendReadwiseRequest[ReadwiseHighlight, int](ctx, url)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
|
||||
if data.NextPageCursor != 0 {
|
||||
cursor.Cursor = strconv.Itoa(data.NextPageCursor)
|
||||
} else {
|
||||
cursor.Cursor = ""
|
||||
}
|
||||
|
||||
highlights = make([]*synchronizator.Node, 0, len(data.Results))
|
||||
|
||||
for i, document := range data.Results {
|
||||
metadata, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
|
||||
node := synchronizator.NewNode(
|
||||
document.Title,
|
||||
"HIGHLIGHTS",
|
||||
metadata,
|
||||
rawData.Results[i],
|
||||
)
|
||||
highlights = append(highlights, node)
|
||||
}
|
||||
|
||||
payload.Response = highlights
|
||||
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
start := time.Now()
|
||||
|
||||
|
|
@ -167,7 +276,7 @@ func main() {
|
|||
return
|
||||
}
|
||||
|
||||
readwiseReader, err := sync.NewPlatform("readwise_reader", nil, nil)
|
||||
readwise, err := sync.NewPlatform("READWISE", nil, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
|
|
@ -186,14 +295,37 @@ func main() {
|
|||
Timeout: time.Second * 2,
|
||||
}
|
||||
|
||||
collection, err := readwiseReader.GetDefaultCollection()
|
||||
documents, err := readwise.AddCollection("DOCUMENTS", nil, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
cursor := &ReadwiseCursor{}
|
||||
|
||||
ctx := context.WithValue(context.Background(), "readwise-cursor", cursor)
|
||||
|
||||
for {
|
||||
err = collection.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
|
||||
err = documents.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
if cursor.Cursor == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
highlights, err := readwise.AddCollection("HIGHLIGHTS", nil, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
cursor.Cursor = ""
|
||||
for {
|
||||
err = highlights.FetchNodes(ctx, getReadwiseHighlights, pagination, pool_config)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
|
|
|
|||
Reference in a new issue