package main import ( "context" "database/sql" "encoding/json" "fmt" "io" "net/http" "net/url" "time" _ "modernc.org/sqlite" synchronizator "git.alecodes.page/alecodes/synchronizator/pkg" ) const API_TOKEN = "" type ReadwiseCursor struct { Cursor string } type ReadwiseApiResponse struct { Count uint64 `json:"count"` NextPageCursor string `json:"nextPageCursor"` Results []ReadwiseDocument `json:"results"` } type RawReadwiseApiResponse struct { Count uint64 `json:"count"` NextPageCursor string `json:"nextPageCursor"` Results []json.RawMessage `json:"results"` // All ass raw } type ReadwiseDocument struct { Id string `json:"id"` Url string `json:"url"` Title string `json:"title"` // Author string `json:"author"` // Source string `json:"source"` // Category string `json:"category"` Location string `json:"location"` // Tags map[string]string `json:"tags"` // SiteName string `json:"site_name"` // CreatedAt string `json:"created_at"` // UpdatedAt string `json:"updated_at"` // Summary string `json:"summary"` SourceUrl string `json:"source_url"` // Notes string `json:"notes"` // ParentId interface{} `json:"parent_id"` // SavedAt string `json:"saved_at"` // LastMovedAt string `json:"last_moved_at"` } func getReadwiseDocuments( ctx context.Context, pagination synchronizator.Pagination, ) (synchronizator.FetchNodesResponse, error) { payload := synchronizator.FetchNodesResponse{ Pagination: pagination, } cursor, ok := ctx.Value("readwise-cursor").(*ReadwiseCursor) if !ok { return payload, fmt.Errorf("Couldn't retreive cursor from context!") } var documents []*synchronizator.Node params := url.Values{} if cursor.Cursor != "" { params.Add("pageCursor", cursor.Cursor) } url := "https://readwise.io/api/v3/list?" + params.Encode() req, err := http.NewRequest("GET", url, nil) if err != nil { fmt.Println("Error creating request:", err) return payload, err } // Add the authorization header req.Header.Set("Authorization", "Token "+API_TOKEN) client := &http.Client{} resp, err := client.Do(req) if err != nil { return payload, err } body, err := io.ReadAll(resp.Body) resp.Body.Close() var data ReadwiseApiResponse err = json.Unmarshal(body, &data) if err != nil { return payload, err } var rawData RawReadwiseApiResponse err = json.Unmarshal(body, &rawData) if err != nil { return payload, err } cursor.Cursor = data.NextPageCursor documents = make([]*synchronizator.Node, 0, len(data.Results)) for i, document := range data.Results { metadata, err := json.Marshal(document) if err != nil { return payload, err } node := synchronizator.NewNode( document.Title, "DOCUMENT", metadata, rawData.Results[i], ) documents = append(documents, node) } payload.Response = documents return payload, nil } func main() { start := time.Now() defer func() { elapsed := time.Now().Sub(start) fmt.Printf("\n\nExecution time took: %s", elapsed) }() connection, err := sql.Open("sqlite", "readwise.sql") if err != nil { fmt.Println(err) return } defer connection.Close() opts := synchronizator.DefaultOptions // opts.Log_level = synchronizator.DEBUG opts.DANGEROUSLY_DROP_TABLES = true sync, err := synchronizator.New(connection, opts) if err != nil { fmt.Println(err) return } readwiseReader, err := sync.NewPlatform("readwise_reader", nil, nil) if err != nil { fmt.Println(err) return } pagination := synchronizator.StartPagination pagination.Pages = 0 pagination.Total = 100 pagination.Limit = 100 pool_config := &synchronizator.WorkConfig{ AmountOfWorkers: 5, MaxRetries: 1, BaseRetryTime: time.Second * 2, RateLimit: synchronizator.NewRateLimiter(20, time.Minute), Timeout: time.Second * 2, } collection, err := readwiseReader.GetDefaultCollection() cursor := &ReadwiseCursor{} ctx := context.WithValue(context.Background(), "readwise-cursor", cursor) for { err = collection.FetchNodes(ctx, getReadwiseDocuments, pagination, pool_config) if err != nil { fmt.Println(err) return } if cursor.Cursor == "" { break } } }