diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6a53ac5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +# ---> VirtualEnv +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json + +# ---> GPG +secring.* + +.env +Dockerfile +.forgejo +.git diff --git a/.forgejo/workflows/build-docker-image.yaml b/.forgejo/workflows/build-docker-image.yaml new file mode 100644 index 0000000..749c3cf --- /dev/null +++ b/.forgejo/workflows/build-docker-image.yaml @@ -0,0 +1,29 @@ +name: Publish image +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + create-docker-images: + runs-on: host + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: git.alecodes.page + username: ${{ vars.CONTAINER_REGISTRY_USER }} + password: ${{ secrets.CONTAINER_REGISTRY_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: | + git.alecodes.page/alecodes/miniflux-archive:latest + git.alecodes.page/alecodes/miniflux-archive:${{ github.sha }} diff --git a/.forgejo/workflows/create.yml b/.forgejo/workflows/create.yml deleted file mode 100644 index 0c619c8..0000000 --- a/.forgejo/workflows/create.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Template migration - -# The workflow will run only when `use this template` is used -on: create - -jobs: - create: - runs-on: docker - - steps: - - name: "Check out the repo" - uses: "actions/checkout@v4" - - - name: "Update commit" - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "chore: initial commit" - commit_options: "--amend" - push_options: "--force" - skip_fetch: true diff --git a/.justfile b/.justfile index c8ecec6..644b56a 100644 --- a/.justfile +++ b/.justfile @@ -1,2 +1,5 @@ run: - go run main.go run linkding + go run main.go run linkding --archive-starred + +docker-build: + docker build . --tag miniflux-archiver:latest diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..53d5b2f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog +All notable changes to this project will be documented in this file. See [conventional commits](https://www.conventionalcommits.org/) for commit guidelines. + +- - - +## 0.1.0 - 2024-12-20 +#### Build system +- add docker build setup - (66b21da) - aleidk +#### Features +- archive read entries - (1788241) - aleidk +- add basic linkding connection - (43f6340) - aleidk +- list entries to archive from miniflux - (43cd24e) - aleidk +- add basic miniflux conection - (22b4abf) - aleidk +- add basic command structure - (1459abd) - aleidk +- first commit - (c8a37ed) - alecodes +#### Miscellaneous Chores +- add cocogitto config - (0a32e22) - aleidk + +- - - + +Changelog generated by [cocogitto](https://github.com/cocogitto/cocogitto). \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..55f951f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM golang:1.23-alpine AS builder + +WORKDIR /usr/src/app + +# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change +COPY go.mod go.sum ./ +RUN go mod download && go mod verify + +COPY . . +RUN go build -v ./main.go + +FROM alpine + +COPY --from=builder /usr/src/app/main /usr/bin/miniflux-archiver + +ENV MFA_CRON="* * * * *" + +WORKDIR /app + +COPY ./entrypoint.sh . + +ENTRYPOINT ["./entrypoint.sh"] + +RUN chmod +x entrypoint.sh /usr/bin/miniflux-archiver + +CMD ["--help"] diff --git a/cmd/archive.go b/cmd/archive.go index 86e1d1c..43cc75b 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -8,17 +8,24 @@ import ( "github.com/spf13/viper" "git.alecodes.page/alecodes/miniflux-archiver/internal/app" + "git.alecodes.page/alecodes/miniflux-archiver/internal/logger" "git.alecodes.page/alecodes/miniflux-archiver/internal/miniflux" + "git.alecodes.page/alecodes/miniflux-archiver/internal/service" ) var Services = []string{ - "linkding", + string(service.ServiceLinkding), } var ( - service_host string - service_user string - service_token string + service_host string + service_user string + service_token string + service_max_requests uint8 + + archive_seen bool + archive_starred bool + archive_method string ) // archiveCmd represents the archive command @@ -29,33 +36,56 @@ var archiveCmd = &cobra.Command{ Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), ValidArgs: Services, Run: func(cmd *cobra.Command, args []string) { - service := args[0] + serviceOption := service.ServiceOption(args[0]) - serviceConfig := app.ServiceConfig{ - Service: service, - Host: service_host, - User: service_user, - Token: service_token, + serviceConfig := service.ServiceConfig{ + Service: serviceOption, + Host: viper.GetString("service_host"), + User: viper.GetString("service_user"), + Token: viper.GetString("service_token"), + Method: service.ServiceArchiveMethod(archive_method), + MaxRequests: service_max_requests, } minifluxConfig := miniflux.MinifluxConfig{ - Host: viper.GetString("miniflux_host"), - Token: viper.GetString("miniflux_token"), - FeedId: viper.GetInt64("miniflux_feed_id"), + Host: viper.GetString("miniflux_host"), + Token: viper.GetString("miniflux_token"), + FeedId: viper.GetInt64("miniflux_feed_id"), + FeedFilter: &miniflux.Filter{}, } - app.Archive(minifluxConfig, serviceConfig) + if archive_seen { + logger.Info("Archiving Feed %v entries", minifluxConfig.FeedId) + app.Archive(minifluxConfig, serviceConfig) + } + + if archive_starred { + logger.Info("Archiving All starred entries") + minifluxConfig.FeedFilter.Starred = "true" + minifluxConfig.FeedId = -1 + app.Archive(minifluxConfig, serviceConfig) + } }, } func init() { rootCmd.AddCommand(archiveCmd) + archiveCmd.Flags(). + BoolVarP(&archive_seen, "archive-seen", "s", true, "If the seen entries should be archived") + archiveCmd.Flags(). + BoolVarP(&archive_starred, "archive-starred", "S", false, "If the starred entries should be archived") + archiveCmd.Flags(). + StringVarP(&archive_method, "archive-method", "m", "seen", "What action to apply to the entries, possible values are: seen, archive, both") + archiveCmd.Flags().StringVar(&service_host, "service-host", "", "127.0.0.1") archiveCmd.Flags().StringVar(&service_user, "service-user", "", "john.doe@mail.cl") archiveCmd.Flags().StringVar(&service_token, "service-token", "", "XXX-XXX-XXX") + archiveCmd.Flags(). + Uint8Var(&service_max_requests, "service_max_requests", 5, "Maximum alowed of concurrent requests") viper.BindPFlag("service_host", archiveCmd.Flags().Lookup("service-host")) viper.BindPFlag("service_host", archiveCmd.Flags().Lookup("service-host")) viper.BindPFlag("service_token", archiveCmd.Flags().Lookup("service-token")) + viper.BindPFlag("service_max_requests", archiveCmd.Flags().Lookup("service-max-requests")) } diff --git a/cog.toml b/cog.toml new file mode 100644 index 0000000..f258492 --- /dev/null +++ b/cog.toml @@ -0,0 +1,25 @@ +from_latest_tag = false +ignore_merge_commits = true +disable_changelog = false +disable_bump_commit = false +generate_mono_repository_global_tag = true +generate_mono_repository_package_tags = true +branch_whitelist = [] +skip_ci = "[skip ci]" +skip_untracked = false +pre_bump_hooks = [] +post_bump_hooks = [] +pre_package_bump_hooks = [] +post_package_bump_hooks = [] + +[git_hooks] + +[commit_types] + +[changelog] +path = "CHANGELOG.md" +authors = [] + +[bump_profiles] + +[packages] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..7d31953 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +echo "$MFA_CRON /usr/bin/miniflux-archiver $*" | crontab - + +crond -f -d 8 diff --git a/go.mod b/go.mod index 3a8addd..d9dcd11 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.23.3 require ( github.com/joho/godotenv v1.5.1 + github.com/piero-vic/go-linkding v0.2.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 + golang.org/x/sync v0.9.0 miniflux.app/v2 v2.2.3 ) diff --git a/go.sum b/go.sum index 054ac6a..b901c3f 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/piero-vic/go-linkding v0.2.0 h1:1LNIeWvYe8Kd+kaG4hsWzYH/vvQ9OaJOMoV+XvXaH+Q= +github.com/piero-vic/go-linkding v0.2.0/go.mod h1:PuwOySAQYmbq4cIDAG1bXDMQwBUuorRkbjM43RdUhao= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -65,6 +67,8 @@ go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= diff --git a/internal/app/archive.go b/internal/app/archive.go index 48c8f02..857db3e 100644 --- a/internal/app/archive.go +++ b/internal/app/archive.go @@ -1,23 +1,52 @@ package app import ( - "fmt" + "context" "git.alecodes.page/alecodes/miniflux-archiver/internal/logger" "git.alecodes.page/alecodes/miniflux-archiver/internal/miniflux" + "git.alecodes.page/alecodes/miniflux-archiver/internal/service" + "golang.org/x/sync/semaphore" ) -func Archive(minifluxConfig miniflux.MinifluxConfig, serviceConfig ServiceConfig) { +func Archive(minifluxConfig miniflux.MinifluxConfig, serviceConfig service.ServiceConfig) { + ctx := context.Background() + mf, err := miniflux.NewMiniflux(minifluxConfig) if err != nil { logger.Fatal("Could not connect to the miniflux server: %v", err) } + externalService, err := service.ResolveService(serviceConfig) + if err != nil { + logger.Fatal(err.Error()) + } + result, err := mf.GetEntries() if err != nil { logger.Fatal("Could not retrieve entries from the miniflux feed: %v", err) } - entry := result.Entries[0] - fmt.Println(entry.Title, entry.Status, entry.Tags) + sem := semaphore.NewWeighted(int64(serviceConfig.MaxRequests)) + + for _, entry := range result.Entries { + if err := sem.Acquire(ctx, 1); err != nil { + logger.Fatal("Failed to acquire semaphore: %v", err) + } + + go func() { + defer sem.Release(1) + + err := externalService.Archive(entry.URL) + if err != nil { + logger.Fatal("Could not archive url \"%v\" from the service: %v", entry.URL, err) + } + + logger.Info("Url \"%v\" has been marked as read", entry.URL) + }() + } + + if err := sem.Acquire(ctx, int64(serviceConfig.MaxRequests)); err != nil { + logger.Fatal("Failed to acquire semaphore: %v", err) + } } diff --git a/internal/app/service.go b/internal/app/service.go deleted file mode 100644 index cf3a046..0000000 --- a/internal/app/service.go +++ /dev/null @@ -1,8 +0,0 @@ -package app - -type ServiceConfig struct { - Service string - Host string - User string - Token string -} diff --git a/internal/miniflux/miniflux.go b/internal/miniflux/miniflux.go index 66a4842..8ad9e46 100644 --- a/internal/miniflux/miniflux.go +++ b/internal/miniflux/miniflux.go @@ -5,11 +5,14 @@ import ( mfApi "miniflux.app/v2/client" ) +type Filter = mfApi.Filter + type MinifluxConfig struct { - Host string - User string - Token string - FeedId int64 + Host string + User string + Token string + FeedId int64 + FeedFilter *Filter } type Miniflux struct { @@ -18,11 +21,13 @@ type Miniflux struct { } func (mf *Miniflux) GetEntries() (*mfApi.EntryResultSet, error) { - filter := &mfApi.Filter{ - Statuses: []string{mfApi.EntryStatusRead, mfApi.EntryStatusRemoved}, - } + mf.FeedFilter.Statuses = []string{mfApi.EntryStatusRead, mfApi.EntryStatusRemoved} - return mf.client.FeedEntries(mf.FeedId, filter) + if mf.FeedId == -1 { + return mf.client.Entries(mf.FeedFilter) + } else { + return mf.client.FeedEntries(mf.FeedId, mf.FeedFilter) + } } func NewMiniflux(config MinifluxConfig) (*Miniflux, error) { diff --git a/internal/service/linkding.go b/internal/service/linkding.go new file mode 100644 index 0000000..53f23b9 --- /dev/null +++ b/internal/service/linkding.go @@ -0,0 +1,93 @@ +package service + +import ( + "encoding/json" + "fmt" + "net/http" + + ldApi "github.com/piero-vic/go-linkding" +) + +type Linkding struct { + ServiceConfig + client *ldApi.Client +} + +func (ld *Linkding) CheckBookmark(url string) (*ldApi.Bookmark, error) { + bookmark := &ldApi.Bookmark{} + + req, err := http.NewRequest( + http.MethodGet, + fmt.Sprintf("%v/api/bookmarks/check/?url=%d", ld.Host, url), + nil, + ) + if err != nil { + return bookmark, err + } + + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Token %s", ld.Token)) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return bookmark, err + } + + defer res.Body.Close() + + if err := json.NewDecoder(res.Body).Decode(bookmark); err != nil { + return bookmark, err + } + + return bookmark, nil +} + +func (ld *Linkding) Archive(url string) error { + bookmark, err := ld.CheckBookmark(url) + + payload := ldApi.CreateBookmarkRequest{ + URL: url, + Unread: false, + TagNames: bookmark.TagNames, + } + + switch ld.ServiceConfig.Method { + case ServiceArchiveMethodArchive: + payload.IsArchived = true + case ServiceArchiveMethodSeen: + payload.Unread = false + case ServiceArchiveMethodBoth: + payload.Unread = false + payload.IsArchived = true + default: + return fmt.Errorf("Archive method is invalid") + } + + if payload.TagNames == nil { + payload.TagNames = []string{} + } + + if err != nil { + _, err = ld.client.UpdateBookmark(bookmark.ID, payload) + } else { + _, err = ld.client.CreateBookmark(payload) + } + + return err +} + +func (ld *Linkding) IsAvailable() (bool, error) { + _, err := ld.client.ListTags(ldApi.ListTagsParams{}) + + return err == nil, err +} + +func NewLinkding(config ServiceConfig) (*Linkding, error) { + mf := &Linkding{ + ServiceConfig: config, + client: ldApi.NewClient(config.Host, config.Token), + } + + return mf, nil +} diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 0000000..f38eec6 --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,54 @@ +package service + +import ( + "fmt" +) + +type ServiceOption string + +const ( + ServiceLinkding ServiceOption = "linkding" +) + +type ServiceArchiveMethod string + +const ( + ServiceArchiveMethodSeen = "seen" + ServiceArchiveMethodArchive = "archive" + ServiceArchiveMethodBoth = "both" +) + +type ServiceConfig struct { + Service ServiceOption + Host string + User string + Token string + Method ServiceArchiveMethod + MaxRequests uint8 +} + +type Service interface { + IsAvailable() (bool, error) + Archive(string) error +} + +func ResolveService(serviceConfig ServiceConfig) (Service, error) { + var service Service + switch serviceConfig.Service { + case ServiceLinkding: + service, _ = NewLinkding(serviceConfig) + default: + return nil, fmt.Errorf("Could not determine service to connect to") + } + + if isAvailable, err := service.IsAvailable(); !isAvailable { + return nil, fmt.Errorf( + "Could not connect to the service %v in %v: %v", + serviceConfig.Service, + serviceConfig.Host, + err, + ) + } + + return service, nil +}