diff --git a/examples/mock_data/Readwise/Document List.bru b/examples/mock_data/Readwise/Document List.bru index f283088..a2138f5 100644 --- a/examples/mock_data/Readwise/Document List.bru +++ b/examples/mock_data/Readwise/Document List.bru @@ -7,5 +7,5 @@ meta { get { url: https://readwise.io/api/v3/list/ body: none - auth: none + auth: inherit } diff --git a/examples/mock_data/Readwise/Highlights export.bru b/examples/mock_data/Readwise/Highlights export.bru new file mode 100644 index 0000000..d3a5125 --- /dev/null +++ b/examples/mock_data/Readwise/Highlights export.bru @@ -0,0 +1,15 @@ +meta { + name: Highlights export + type: http + seq: 3 +} + +get { + url: https://readwise.io/api/v3/list/?withHtmlContent=true + body: none + auth: inherit +} + +params:query { + withHtmlContent: true +} diff --git a/examples/mock_data/Readwise/collection.bru b/examples/mock_data/Readwise/collection.bru new file mode 100644 index 0000000..35276a3 --- /dev/null +++ b/examples/mock_data/Readwise/collection.bru @@ -0,0 +1,9 @@ +auth { + mode: apikey +} + +auth:apikey { + key: Authorization + value: Token {{API_KEY}} + placement: header +} \ No newline at end of file diff --git a/examples/mock_data/Readwise/environments/Readwise.bru b/examples/mock_data/Readwise/environments/Readwise.bru index d9d33b2..8958398 100644 --- a/examples/mock_data/Readwise/environments/Readwise.bru +++ b/examples/mock_data/Readwise/environments/Readwise.bru @@ -1,3 +1,3 @@ vars:secret [ - API-KEY + API_KEY ] diff --git a/examples/readwise/main.go b/examples/readwise/main.go index bb8f70b..b85c660 100644 --- a/examples/readwise/main.go +++ b/examples/readwise/main.go @@ -22,6 +22,7 @@ type ReadwiseCursor struct { } type ReadwiseApiResponse struct { + Detail string `json:detail` Count uint64 `json:"count"` NextPageCursor string `json:"nextPageCursor"` Results []ReadwiseDocument `json:"results"` @@ -70,6 +71,7 @@ func getReadwiseDocuments( var documents []*synchronizator.Node params := url.Values{} + params.Add("withHtmlContent", "true") if cursor.Cursor != "" { params.Add("pageCursor", cursor.Cursor) } @@ -99,6 +101,14 @@ func getReadwiseDocuments( return payload, err } + if resp.StatusCode > 201 { + return payload, fmt.Errorf( + "Request failed with status %v: %v", + resp.StatusCode, + data.Detail, + ) + } + var rawData RawReadwiseApiResponse err = json.Unmarshal(body, &rawData) if err != nil { @@ -164,14 +174,15 @@ func main() { } pagination := synchronizator.StartPagination - pagination.Pages = 0 + pagination.Pages = 1 + pagination.Offset = 100 pagination.Total = 100 pagination.Limit = 100 pool_config := &synchronizator.WorkConfig{ AmountOfWorkers: 5, - MaxRetries: 1, - BaseRetryTime: time.Second * 2, - RateLimit: synchronizator.NewRateLimiter(20, time.Minute), + MaxRetries: 2, + BaseRetryTime: time.Second * 30, + RateLimit: synchronizator.NewRateLimiter(10, time.Minute), Timeout: time.Second * 2, } diff --git a/pkg/collection.go b/pkg/collection.go index a033936..bcb4678 100644 --- a/pkg/collection.go +++ b/pkg/collection.go @@ -114,7 +114,7 @@ func (collection *Collection) FetchNodes( collection.childs = slices.Concat(collection.childs, values) - fmt.Printf("Nodes: %v\n", len(collection.childs)) + fmt.Printf("Nodes added: %d, Nodes in collection: %d\n", len(values), len(collection.childs)) err = BulkCreateNode(collection._conn, values) if err != nil { diff --git a/pkg/fetcher.go b/pkg/fetcher.go index 058a43d..0d0e8e8 100644 --- a/pkg/fetcher.go +++ b/pkg/fetcher.go @@ -133,7 +133,6 @@ func fetchWithPagination[T any]( current_page += page_offset for current_page <= start_pagination.Pages { - fmt.Printf("Total pages: %v, Current page: %v\n", start_pagination.Pages, current_page) page := start_pagination page.Offset = current_page * page.Limit tasks <- page diff --git a/pkg/workpool.go b/pkg/workpool.go index 58d4cc1..d8fdc7f 100644 --- a/pkg/workpool.go +++ b/pkg/workpool.go @@ -240,7 +240,6 @@ func workUnitDispatcher[T, S any]( config.TasksProcessed.Add(1) case <-ctx.Done(): - fmt.Println("context done") return } }