generated from alecodes/base-template
fead: expand archive options
add an option to select how to archive the entries add an option for what entries to archive change the concurrentcy limit to a semaphore instead of a ticker
This commit is contained in:
parent
1788241cd5
commit
b0506c3eab
9 changed files with 99 additions and 72 deletions
|
|
@ -1,2 +1,2 @@
|
|||
run:
|
||||
go run main.go run linkding
|
||||
go run main.go run linkding --archive-starred
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/spf13/viper"
|
||||
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/app"
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/logger"
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/miniflux"
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/service"
|
||||
)
|
||||
|
|
@ -20,6 +21,11 @@ var (
|
|||
service_host string
|
||||
service_user string
|
||||
service_token string
|
||||
service_max_requests uint8
|
||||
|
||||
archive_seen bool
|
||||
archive_starred bool
|
||||
archive_method string
|
||||
)
|
||||
|
||||
// archiveCmd represents the archive command
|
||||
|
|
@ -37,26 +43,49 @@ var archiveCmd = &cobra.Command{
|
|||
Host: viper.GetString("service_host"),
|
||||
User: viper.GetString("service_user"),
|
||||
Token: viper.GetString("service_token"),
|
||||
Method: service.ServiceArchiveMethod(archive_method),
|
||||
MaxRequests: service_max_requests,
|
||||
}
|
||||
|
||||
minifluxConfig := miniflux.MinifluxConfig{
|
||||
Host: viper.GetString("miniflux_host"),
|
||||
Token: viper.GetString("miniflux_token"),
|
||||
FeedId: viper.GetInt64("miniflux_feed_id"),
|
||||
FeedFilter: &miniflux.Filter{},
|
||||
}
|
||||
|
||||
if archive_seen {
|
||||
logger.Info("Archiving Feed %v entries", minifluxConfig.FeedId)
|
||||
app.Archive(minifluxConfig, serviceConfig)
|
||||
}
|
||||
|
||||
if archive_starred {
|
||||
logger.Info("Archiving All starred entries")
|
||||
minifluxConfig.FeedFilter.Starred = "true"
|
||||
minifluxConfig.FeedId = -1
|
||||
app.Archive(minifluxConfig, serviceConfig)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(archiveCmd)
|
||||
|
||||
archiveCmd.Flags().
|
||||
BoolVarP(&archive_seen, "archive-seen", "s", true, "If the seen entries should be archived")
|
||||
archiveCmd.Flags().
|
||||
BoolVarP(&archive_starred, "archive-starred", "S", false, "If the starred entries should be archived")
|
||||
archiveCmd.Flags().
|
||||
StringVarP(&archive_method, "archive-method", "m", "seen", "What action to apply to the entries, possible values are: seen, archive, both")
|
||||
|
||||
archiveCmd.Flags().StringVar(&service_host, "service-host", "", "127.0.0.1")
|
||||
archiveCmd.Flags().StringVar(&service_user, "service-user", "", "john.doe@mail.cl")
|
||||
archiveCmd.Flags().StringVar(&service_token, "service-token", "", "XXX-XXX-XXX")
|
||||
archiveCmd.Flags().
|
||||
Uint8Var(&service_max_requests, "service_max_requests", 5, "Maximum alowed of concurrent requests")
|
||||
|
||||
viper.BindPFlag("service_host", archiveCmd.Flags().Lookup("service-host"))
|
||||
viper.BindPFlag("service_host", archiveCmd.Flags().Lookup("service-host"))
|
||||
viper.BindPFlag("service_token", archiveCmd.Flags().Lookup("service-token"))
|
||||
viper.BindPFlag("service_max_requests", archiveCmd.Flags().Lookup("service-max-requests"))
|
||||
}
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -4,8 +4,10 @@ go 1.23.3
|
|||
|
||||
require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/piero-vic/go-linkding v0.2.0
|
||||
github.com/spf13/cobra v1.8.1
|
||||
github.com/spf13/viper v1.19.0
|
||||
golang.org/x/sync v0.9.0
|
||||
miniflux.app/v2 v2.2.3
|
||||
)
|
||||
|
||||
|
|
@ -16,7 +18,6 @@ require (
|
|||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/piero-vic/go-linkding v0.2.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -67,6 +67,8 @@ go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
|||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
|
||||
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
|
||||
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
|
||||
|
|
|
|||
|
|
@ -1,35 +1,17 @@
|
|||
package app
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
"context"
|
||||
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/logger"
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/miniflux"
|
||||
"git.alecodes.page/alecodes/miniflux-archiver/internal/service"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
// NewRateLimiter creates a rate limiter that emits time events at a specified rate.
|
||||
// request_per specifies the number of requests allowed per time_scale duration.
|
||||
// time_scale specifies the duration over which the requests are allowed.
|
||||
func newRateLimiter(request_per int, time_scale time.Duration) <-chan time.Time {
|
||||
rate_limit := make(chan time.Time, request_per)
|
||||
tickrate := time_scale / time.Duration(request_per)
|
||||
|
||||
for range request_per {
|
||||
rate_limit <- time.Now()
|
||||
}
|
||||
|
||||
go func() {
|
||||
for t := range time.Tick(tickrate) {
|
||||
rate_limit <- t
|
||||
}
|
||||
}()
|
||||
|
||||
return rate_limit
|
||||
}
|
||||
|
||||
func Archive(minifluxConfig miniflux.MinifluxConfig, serviceConfig service.ServiceConfig) {
|
||||
ctx := context.Background()
|
||||
|
||||
mf, err := miniflux.NewMiniflux(minifluxConfig)
|
||||
if err != nil {
|
||||
logger.Fatal("Could not connect to the miniflux server: %v", err)
|
||||
|
|
@ -45,17 +27,16 @@ func Archive(minifluxConfig miniflux.MinifluxConfig, serviceConfig service.Servi
|
|||
logger.Fatal("Could not retrieve entries from the miniflux feed: %v", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ticker := newRateLimiter(5, 10*time.Second)
|
||||
sem := semaphore.NewWeighted(int64(serviceConfig.MaxRequests))
|
||||
|
||||
for _, entry := range result.Entries {
|
||||
wg.Add(1)
|
||||
if err := sem.Acquire(ctx, 1); err != nil {
|
||||
logger.Fatal("Failed to acquire semaphore: %v", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-ticker
|
||||
defer sem.Release(1)
|
||||
|
||||
// logger.Debug("Sending url \"%v\" to be marked as read", entry.URL)
|
||||
err := externalService.Archive(entry.URL)
|
||||
if err != nil {
|
||||
logger.Fatal("Could not archive url \"%v\" from the service: %v", entry.URL, err)
|
||||
|
|
@ -65,5 +46,7 @@ func Archive(minifluxConfig miniflux.MinifluxConfig, serviceConfig service.Servi
|
|||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
if err := sem.Acquire(ctx, int64(serviceConfig.MaxRequests)); err != nil {
|
||||
logger.Fatal("Failed to acquire semaphore: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
package config
|
||||
|
||||
import "git.alecodes.page/alecodes/miniflux-archiver/internal/service"
|
||||
|
||||
type MinifluxConfig struct {
|
||||
Host string
|
||||
User string
|
||||
Token string
|
||||
FeedId int64
|
||||
}
|
||||
|
||||
type ServiceConfig struct {
|
||||
Service service.ServiceOption
|
||||
Host string
|
||||
User string
|
||||
Token string
|
||||
}
|
||||
|
|
@ -5,11 +5,14 @@ import (
|
|||
mfApi "miniflux.app/v2/client"
|
||||
)
|
||||
|
||||
type Filter = mfApi.Filter
|
||||
|
||||
type MinifluxConfig struct {
|
||||
Host string
|
||||
User string
|
||||
Token string
|
||||
FeedId int64
|
||||
FeedFilter *Filter
|
||||
}
|
||||
|
||||
type Miniflux struct {
|
||||
|
|
@ -18,11 +21,13 @@ type Miniflux struct {
|
|||
}
|
||||
|
||||
func (mf *Miniflux) GetEntries() (*mfApi.EntryResultSet, error) {
|
||||
filter := &mfApi.Filter{
|
||||
Statuses: []string{mfApi.EntryStatusRead, mfApi.EntryStatusRemoved},
|
||||
}
|
||||
mf.FeedFilter.Statuses = []string{mfApi.EntryStatusRead, mfApi.EntryStatusRemoved}
|
||||
|
||||
return mf.client.FeedEntries(mf.FeedId, filter)
|
||||
if mf.FeedId == -1 {
|
||||
return mf.client.Entries(mf.FeedFilter)
|
||||
} else {
|
||||
return mf.client.FeedEntries(mf.FeedId, mf.FeedFilter)
|
||||
}
|
||||
}
|
||||
|
||||
func NewMiniflux(config MinifluxConfig) (*Miniflux, error) {
|
||||
|
|
|
|||
|
|
@ -52,6 +52,18 @@ func (ld *Linkding) Archive(url string) error {
|
|||
TagNames: bookmark.TagNames,
|
||||
}
|
||||
|
||||
switch ld.ServiceConfig.Method {
|
||||
case ServiceArchiveMethodArchive:
|
||||
payload.IsArchived = true
|
||||
case ServiceArchiveMethodSeen:
|
||||
payload.Unread = false
|
||||
case ServiceArchiveMethodBoth:
|
||||
payload.Unread = false
|
||||
payload.IsArchived = true
|
||||
default:
|
||||
return fmt.Errorf("Archive method is invalid")
|
||||
}
|
||||
|
||||
if payload.TagNames == nil {
|
||||
payload.TagNames = []string{}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package service
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type ServiceOption string
|
||||
|
||||
|
|
@ -8,11 +10,21 @@ const (
|
|||
ServiceLinkding ServiceOption = "linkding"
|
||||
)
|
||||
|
||||
type ServiceArchiveMethod string
|
||||
|
||||
const (
|
||||
ServiceArchiveMethodSeen = "seen"
|
||||
ServiceArchiveMethodArchive = "archive"
|
||||
ServiceArchiveMethodBoth = "both"
|
||||
)
|
||||
|
||||
type ServiceConfig struct {
|
||||
Service ServiceOption
|
||||
Host string
|
||||
User string
|
||||
Token string
|
||||
Method ServiceArchiveMethod
|
||||
MaxRequests uint8
|
||||
}
|
||||
|
||||
type Service interface {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue