From 598a389e3dd90fc81c7207e8ad8ca3ed816c8bd3 Mon Sep 17 00:00:00 2001 From: Harvey Tindall Date: Sat, 29 Nov 2025 14:13:34 +0000 Subject: [PATCH] jellyseerr: fix extremely long import, run only once cache was being invalidated for every user, and on my 5000 user test instance, this sweated jellyseerr and my computer (audibly). Also, since this only needs to realistically run once, a flag is set in the database to indicate it's been done, and unset once the feature is disabled. It'll only run on boot if the flag is unset, or if triggered by the /tasks route. Will likely add manual trigger buttons on the web as well. --- config.go | 1 + config/config-base.yaml | 1 + jellyseerr-d.go | 25 ++++++++++++- jellyseerr/jellyseerr.go | 81 ++++++++++++++++++++++++++++++++-------- main.go | 13 +++++-- migrations.go | 9 +++++ ts/modules/accounts.ts | 6 +++ 7 files changed, 116 insertions(+), 20 deletions(-) diff --git a/config.go b/config.go index 44a198f..d32035b 100644 --- a/config.go +++ b/config.go @@ -416,6 +416,7 @@ func (config *Config) ReloadDependents(app *appContext) { } app.email = NewEmailer(config, app.storage, app.LoggerSet) + } func (app *appContext) ReloadConfig() { diff --git a/config/config-base.yaml b/config/config-base.yaml index 0c96547..03afac7 100644 --- a/config/config-base.yaml +++ b/config/config-base.yaml @@ -1407,6 +1407,7 @@ sections: depends_true: enabled description: Existing users (and those created outside jfa-go) will have their contact info imported to Jellyseerr. + deprecated: true - setting: constraints_note name: 'Unique Emails:' type: note diff --git a/jellyseerr-d.go b/jellyseerr-d.go index 78d4ff6..4544350 100644 --- a/jellyseerr-d.go +++ b/jellyseerr-d.go @@ -9,8 +9,13 @@ import ( lm "github.com/hrfee/jfa-go/logmessages" ) +type JellyseerrInitialSyncStatus struct { + Done bool +} + +// Ensure the Jellyseerr cache is up to date before calling. func (app *appContext) SynchronizeJellyseerrUser(jfID string) { - user, imported, err := app.js.GetOrImportUser(jfID) + user, imported, err := app.js.GetOrImportUser(jfID, true) if err != nil { app.debug.Printf(lm.FailedImportUser, lm.Jellyseerr, jfID, err) return @@ -63,19 +68,30 @@ func (app *appContext) SynchronizeJellyseerrUser(jfID string) { } func (app *appContext) SynchronizeJellyseerrUsers() { + jsSync := JellyseerrInitialSyncStatus{} + app.storage.db.Get("jellyseerr_inital_sync_status", &jsSync) + if jsSync.Done { + return + } + users, err := app.jf.GetUsers(false) if err != nil { app.err.Printf(lm.FailedGetUsers, lm.Jellyfin, err) return } + app.js.ReloadCache() // I'm sure Jellyseerr can handle it, // but past issues with the Jellyfin db scare me from // running these concurrently. W/e, its a bg task anyway. for _, user := range users { app.SynchronizeJellyseerrUser(user.ID) } + // Don't run again until this flag is unset + // Stored in the DB as it's not something the user needs to see. + app.storage.db.Upsert("jellyseerr_inital_sync_status", JellyseerrInitialSyncStatus{true}) } +// Not really a normal daemon, since it'll only fire once when the feature is enabled. func newJellyseerrDaemon(interval time.Duration, app *appContext) *GenericDaemon { d := NewGenericDaemon(interval, app, func(app *appContext) { @@ -83,5 +99,12 @@ func newJellyseerrDaemon(interval time.Duration, app *appContext) *GenericDaemon }, ) d.Name("Jellyseerr import") + + jsSync := JellyseerrInitialSyncStatus{} + app.storage.db.Get("jellyseerr_inital_sync_status", &jsSync) + if jsSync.Done { + return nil + } + return d } diff --git a/jellyseerr/jellyseerr.go b/jellyseerr/jellyseerr.go index 7415b94..2e35d6e 100644 --- a/jellyseerr/jellyseerr.go +++ b/jellyseerr/jellyseerr.go @@ -25,7 +25,9 @@ type Jellyseerr struct { server, key string header map[string]string httpClient *http.Client - userCache map[string]User // Map of jellyfin IDs to users + userCache map[string]User // Map of jellyfin IDs to users + jsToJfID map[int64]string // Map of jellyseerr IDs to jellyfin IDs + invalidatedUsers map[int64]bool // Map of jellyseerr IDs needing a re-caching cacheExpiry time.Time cacheLength time.Duration timeoutHandler co.TimeoutHandler @@ -51,6 +53,8 @@ func NewJellyseerr(server, key string, timeoutHandler co.TimeoutHandler) *Jellys cacheExpiry: time.Now(), timeoutHandler: timeoutHandler, userCache: map[string]User{}, + jsToJfID: map[int64]string{}, + invalidatedUsers: map[int64]bool{}, LogRequestBodies: false, } } @@ -158,6 +162,7 @@ func (js *Jellyseerr) ImportFromJellyfin(jfIDs ...string) ([]User, error) { for _, u := range data { if u.JellyfinUserID != "" { js.userCache[u.JellyfinUserID] = u + js.jsToJfID[u.ID] = u.JellyfinUserID } } return data, err @@ -166,8 +171,13 @@ func (js *Jellyseerr) ImportFromJellyfin(jfIDs ...string) ([]User, error) { func (js *Jellyseerr) getUsers() error { if js.cacheExpiry.After(time.Now()) { return nil + if len(js.invalidatedUsers) != 0 { + return js.getInvalidatedUsers() + } } js.cacheExpiry = time.Now().Add(js.cacheLength) + userCache := map[string]User{} + jsToJfID := map[int64]string{} pageCount := 1 pageIndex := 0 for { @@ -179,7 +189,8 @@ func (js *Jellyseerr) getUsers() error { if u.JellyfinUserID == "" { continue } - js.userCache[u.JellyfinUserID] = u + userCache[u.JellyfinUserID] = u + jsToJfID[u.ID] = u.JellyfinUserID } pageCount = res.Page.Pages pageIndex++ @@ -187,6 +198,10 @@ func (js *Jellyseerr) getUsers() error { break } } + js.userCache = userCache + js.jsToJfID = jsToJfID + js.invalidatedUsers = map[int64]bool{} + return nil } @@ -207,15 +222,15 @@ func (js *Jellyseerr) getUserPage(page int) (GetUsersDTO, error) { } func (js *Jellyseerr) MustGetUser(jfID string) (User, error) { - u, _, err := js.GetOrImportUser(jfID) + u, _, err := js.GetOrImportUser(jfID, false) return u, err } // GetImportedUser provides the same function as ImportFromJellyfin, but will always return the user, // even if they already existed. Also returns whether the user was imported or not, -func (js *Jellyseerr) GetOrImportUser(jfID string) (u User, imported bool, err error) { +func (js *Jellyseerr) GetOrImportUser(jfID string, fixedCache bool) (u User, imported bool, err error) { imported = false - u, err = js.GetExistingUser(jfID) + u, err = js.GetExistingUser(jfID, fixedCache) if err == nil { return } @@ -233,15 +248,24 @@ func (js *Jellyseerr) GetOrImportUser(jfID string) (u User, imported bool, err e return } -func (js *Jellyseerr) GetExistingUser(jfID string) (u User, err error) { +func (js *Jellyseerr) GetExistingUser(jfID string, fixedCache bool) (u User, err error) { js.getUsers() ok := false err = nil - if u, ok = js.userCache[jfID]; ok { + u, ok = js.userCache[jfID] + _, invalidated := js.invalidatedUsers[u.ID] + if ok && !invalidated { return } - js.cacheExpiry = time.Now() - js.getUsers() + if invalidated { + err = js.getInvalidatedUsers() + if err != nil { + return + } + } else if !fixedCache { + js.cacheExpiry = time.Now() + js.getUsers() + } if u, ok = js.userCache[jfID]; ok { err = nil return @@ -254,7 +278,7 @@ func (js *Jellyseerr) getUser(jfID string) (User, error) { if js.AutoImportUsers { return js.MustGetUser(jfID) } - return js.GetExistingUser(jfID) + return js.GetExistingUser(jfID, false) } func (js *Jellyseerr) Me() (User, error) { @@ -268,6 +292,25 @@ func (js *Jellyseerr) Me() (User, error) { return data, err } +func (js *Jellyseerr) getInvalidatedUsers() error { + // FIXME: Collect errors and return + for jellyseerrID, _ := range js.invalidatedUsers { + jfID, ok := js.jsToJfID[jellyseerrID] + if !ok { + continue + } + user, err := js.UserByID(jellyseerrID) + if err != nil { + continue + } + js.userCache[jfID] = user + js.jsToJfID[jellyseerrID] = jfID + delete(js.invalidatedUsers, jellyseerrID) + + } + return nil +} + func (js *Jellyseerr) GetPermissions(jfID string) (Permissions, error) { data := permissionsDTO{Permissions: -1} u, err := js.getUser(jfID) @@ -295,6 +338,7 @@ func (js *Jellyseerr) SetPermissions(jfID string, perm Permissions) error { } u.Permissions = perm js.userCache[jfID] = u + js.jsToJfID[u.ID] = jfID return nil } @@ -310,6 +354,7 @@ func (js *Jellyseerr) ApplyTemplateToUser(jfID string, tmpl UserTemplate) error } u.UserTemplate = tmpl js.userCache[jfID] = u + js.jsToJfID[u.ID] = jfID return nil } @@ -326,8 +371,7 @@ func (js *Jellyseerr) ModifyUser(jfID string, conf map[UserField]any) error { if err != nil { return err } - // Lazily just invalidate the cache. - js.cacheExpiry = time.Now() + js.invalidatedUsers[u.ID] = true return nil } @@ -413,12 +457,19 @@ func (js *Jellyseerr) ModifyMainUserSettings(jfID string, conf MainUserSettings) if err != nil { return err } + return js.ModifyMainUserSettingsByID(u.ID, conf) +} - _, _, err = js.post(fmt.Sprintf(js.server+"/user/%d/settings/main", u.ID), conf, false) +func (js *Jellyseerr) ModifyMainUserSettingsByID(jellyseerrID int64, conf MainUserSettings) error { + _, _, err := js.post(fmt.Sprintf(js.server+"/user/%d/settings/main", jellyseerrID), conf, false) if err != nil { return err } - // Lazily just invalidate the cache. - js.cacheExpiry = time.Now() + js.invalidatedUsers[jellyseerrID] = true return nil } + +func (js *Jellyseerr) ReloadCache() error { + js.cacheExpiry = time.Now() + return js.getUsers() +} diff --git a/main.go b/main.go index c71b6ce..968a962 100644 --- a/main.go +++ b/main.go @@ -516,11 +516,16 @@ func start(asDaemon, firstCall bool) { go app.userDaemon.run() defer app.userDaemon.Shutdown() - if app.config.Section("jellyseerr").Key("enabled").MustBool(false) && app.config.Section("jellyseerr").Key("import_existing").MustBool(false) { + if app.config.Section("jellyseerr").Key("enabled").MustBool(false) { + // import_existing_users setting is deprecated, now it'll run when jellyseerr is enabled, or when triggered manually. // jellyseerrDaemon = newJellyseerrDaemon(time.Duration(30*time.Second), app) - app.jellyseerrDaemon = newJellyseerrDaemon(time.Duration(10*time.Minute), app) - go app.jellyseerrDaemon.run() - defer app.jellyseerrDaemon.Shutdown() + app.jellyseerrDaemon = newJellyseerrDaemon(time.Duration(24*time.Hour), app) + if app.jellyseerrDaemon != nil { + go app.jellyseerrDaemon.run() + // Run on startup + go app.jellyseerrDaemon.Trigger() + defer app.jellyseerrDaemon.Shutdown() + } } if app.config.Section("password_resets").Key("enabled").MustBool(false) && serverType == mediabrowser.JellyfinServer { diff --git a/migrations.go b/migrations.go index 9ffef37..d700474 100644 --- a/migrations.go +++ b/migrations.go @@ -22,6 +22,7 @@ func runMigrations(app *appContext) { // migrateHyphens(app) migrateToBadger(app) intialiseCustomContent(app) + migrateJellyseerrImportDaemon(app) } // Migrate pre-0.2.0 user templates to profiles @@ -505,3 +506,11 @@ func migrateExternalURL(app *appContext) { return } } + +// Migrate from use of "Import Existing Users" Jellyseerr import daemon to one-time run when enabling the feature. +func migrateJellyseerrImportDaemon(app *appContext) { + // When Jellyseerr is disabled, set this flag to false so that an initial sync happens the next time it is enabled. + if !(app.config.Section("jellyseerr").Key("enabled").MustBool(false)) { + app.storage.db.Upsert("jellyseerr_inital_sync_status", JellyseerrInitialSyncStatus{false}) + } +} diff --git a/ts/modules/accounts.ts b/ts/modules/accounts.ts index 08c4ccd..2fb9706 100644 --- a/ts/modules/accounts.ts +++ b/ts/modules/accounts.ts @@ -2190,6 +2190,12 @@ export class accountsList extends PaginatedList { this.focusAccount(userID); } +} + +// An alternate view showing accounts in sub-lists grouped by group/label. +export class groupedAccountsList { + + } export const accountURLEvent = (id: string) => { return new CustomEvent(accountsList._accountURLEvent, {"detail": id}) };