jellyseerr: fix extremely long import, run only once

cache was being invalidated for every user, and on my 5000 user test
instance, this sweated jellyseerr and my computer (audibly). Also, since
this only needs to realistically run once, a flag is set in the database
to indicate it's been done, and unset once the feature is disabled.
It'll only run on boot if the flag is unset, or if triggered by the
/tasks route. Will likely add manual trigger buttons on the web as well.

config.go

@@ -416,6 +416,7 @@ func (config *Config) ReloadDependents(app *appContext) {
 	}
 
 	app.email = NewEmailer(config, app.storage, app.LoggerSet)
+
 }
 
 func (app *appContext) ReloadConfig() {

config/config-base.yaml

@@ -1407,6 +1407,7 @@ sections:
     depends_true: enabled
     description: Existing users (and those created outside jfa-go) will have their
       contact info imported to Jellyseerr.
+    deprecated: true
   - setting: constraints_note
     name: 'Unique Emails:'
     type: note

jellyseerr-d.go

@@ -9,8 +9,13 @@ import (
 	lm "github.com/hrfee/jfa-go/logmessages"
 )
 
+type JellyseerrInitialSyncStatus struct {
+	Done bool
+}
+
+// Ensure the Jellyseerr cache is up to date before calling.
 func (app *appContext) SynchronizeJellyseerrUser(jfID string) {
-	user, imported, err := app.js.GetOrImportUser(jfID)
+	user, imported, err := app.js.GetOrImportUser(jfID, true)
 	if err != nil {
 		app.debug.Printf(lm.FailedImportUser, lm.Jellyseerr, jfID, err)
 		return
@@ -63,19 +68,30 @@ func (app *appContext) SynchronizeJellyseerrUser(jfID string) {
 }
 
 func (app *appContext) SynchronizeJellyseerrUsers() {
+	jsSync := JellyseerrInitialSyncStatus{}
+	app.storage.db.Get("jellyseerr_inital_sync_status", &jsSync)
+	if jsSync.Done {
+		return
+	}
+
 	users, err := app.jf.GetUsers(false)
 	if err != nil {
 		app.err.Printf(lm.FailedGetUsers, lm.Jellyfin, err)
 		return
 	}
+	app.js.ReloadCache()
 	// I'm sure Jellyseerr can handle it,
 	// but past issues with the Jellyfin db scare me from
 	// running these concurrently. W/e, its a bg task anyway.
 	for _, user := range users {
 		app.SynchronizeJellyseerrUser(user.ID)
 	}
+	// Don't run again until this flag is unset
+	// Stored in the DB as it's not something the user needs to see.
+	app.storage.db.Upsert("jellyseerr_inital_sync_status", JellyseerrInitialSyncStatus{true})
 }
 
+// Not really a normal daemon, since it'll only fire once when the feature is enabled.
 func newJellyseerrDaemon(interval time.Duration, app *appContext) *GenericDaemon {
 	d := NewGenericDaemon(interval, app,
 		func(app *appContext) {
@@ -83,5 +99,12 @@ func newJellyseerrDaemon(interval time.Duration, app *appContext) *GenericDaemon
 		},
 	)
 	d.Name("Jellyseerr import")
+
+	jsSync := JellyseerrInitialSyncStatus{}
+	app.storage.db.Get("jellyseerr_inital_sync_status", &jsSync)
+	if jsSync.Done {
+		return nil
+	}
+
 	return d
 }

jellyseerr/jellyseerr.go

@@ -25,7 +25,9 @@ type Jellyseerr struct {
 	server, key      string
 	header           map[string]string
 	httpClient       *http.Client
-	userCache        map[string]User // Map of jellyfin IDs to users
+	userCache        map[string]User  // Map of jellyfin IDs to users
+	jsToJfID         map[int64]string // Map of jellyseerr IDs to jellyfin IDs
+	invalidatedUsers map[int64]bool   // Map of jellyseerr IDs needing a re-caching
 	cacheExpiry      time.Time
 	cacheLength      time.Duration
 	timeoutHandler   co.TimeoutHandler
@@ -51,6 +53,8 @@ func NewJellyseerr(server, key string, timeoutHandler co.TimeoutHandler) *Jellys
 		cacheExpiry:      time.Now(),
 		timeoutHandler:   timeoutHandler,
 		userCache:        map[string]User{},
+		jsToJfID:         map[int64]string{},
+		invalidatedUsers: map[int64]bool{},
 		LogRequestBodies: false,
 	}
 }
@@ -158,6 +162,7 @@ func (js *Jellyseerr) ImportFromJellyfin(jfIDs ...string) ([]User, error) {
 	for _, u := range data {
 		if u.JellyfinUserID != "" {
 			js.userCache[u.JellyfinUserID] = u
+			js.jsToJfID[u.ID] = u.JellyfinUserID
 		}
 	}
 	return data, err
@@ -166,8 +171,13 @@ func (js *Jellyseerr) ImportFromJellyfin(jfIDs ...string) ([]User, error) {
 func (js *Jellyseerr) getUsers() error {
 	if js.cacheExpiry.After(time.Now()) {
 		return nil
+		if len(js.invalidatedUsers) != 0 {
+			return js.getInvalidatedUsers()
+		}
 	}
 	js.cacheExpiry = time.Now().Add(js.cacheLength)
+	userCache := map[string]User{}
+	jsToJfID := map[int64]string{}
 	pageCount := 1
 	pageIndex := 0
 	for {
@@ -179,7 +189,8 @@ func (js *Jellyseerr) getUsers() error {
 			if u.JellyfinUserID == "" {
 				continue
 			}
-			js.userCache[u.JellyfinUserID] = u
+			userCache[u.JellyfinUserID] = u
+			jsToJfID[u.ID] = u.JellyfinUserID
 		}
 		pageCount = res.Page.Pages
 		pageIndex++
@@ -187,6 +198,10 @@ func (js *Jellyseerr) getUsers() error {
 			break
 		}
 	}
+	js.userCache = userCache
+	js.jsToJfID = jsToJfID
+	js.invalidatedUsers = map[int64]bool{}
+
 	return nil
 }
 
@@ -207,15 +222,15 @@ func (js *Jellyseerr) getUserPage(page int) (GetUsersDTO, error) {
 }
 
 func (js *Jellyseerr) MustGetUser(jfID string) (User, error) {
-	u, _, err := js.GetOrImportUser(jfID)
+	u, _, err := js.GetOrImportUser(jfID, false)
 	return u, err
 }
 
 // GetImportedUser provides the same function as ImportFromJellyfin, but will always return the user,
 // even if they already existed. Also returns whether the user was imported or not,
-func (js *Jellyseerr) GetOrImportUser(jfID string) (u User, imported bool, err error) {
+func (js *Jellyseerr) GetOrImportUser(jfID string, fixedCache bool) (u User, imported bool, err error) {
 	imported = false
-	u, err = js.GetExistingUser(jfID)
+	u, err = js.GetExistingUser(jfID, fixedCache)
 	if err == nil {
 		return
 	}
@@ -233,15 +248,24 @@ func (js *Jellyseerr) GetOrImportUser(jfID string) (u User, imported bool, err e
 	return
 }
 
-func (js *Jellyseerr) GetExistingUser(jfID string) (u User, err error) {
+func (js *Jellyseerr) GetExistingUser(jfID string, fixedCache bool) (u User, err error) {
 	js.getUsers()
 	ok := false
 	err = nil
-	if u, ok = js.userCache[jfID]; ok {
+	u, ok = js.userCache[jfID]
+	_, invalidated := js.invalidatedUsers[u.ID]
+	if ok && !invalidated {
 		return
 	}
-	js.cacheExpiry = time.Now()
-	js.getUsers()
+	if invalidated {
+		err = js.getInvalidatedUsers()
+		if err != nil {
+			return
+		}
+	} else if !fixedCache {
+		js.cacheExpiry = time.Now()
+		js.getUsers()
+	}
 	if u, ok = js.userCache[jfID]; ok {
 		err = nil
 		return
@@ -254,7 +278,7 @@ func (js *Jellyseerr) getUser(jfID string) (User, error) {
 	if js.AutoImportUsers {
 		return js.MustGetUser(jfID)
 	}
-	return js.GetExistingUser(jfID)
+	return js.GetExistingUser(jfID, false)
 }
 
 func (js *Jellyseerr) Me() (User, error) {
@@ -268,6 +292,25 @@ func (js *Jellyseerr) Me() (User, error) {
 	return data, err
 }
 
+func (js *Jellyseerr) getInvalidatedUsers() error {
+	// FIXME: Collect errors and return
+	for jellyseerrID, _ := range js.invalidatedUsers {
+		jfID, ok := js.jsToJfID[jellyseerrID]
+		if !ok {
+			continue
+		}
+		user, err := js.UserByID(jellyseerrID)
+		if err != nil {
+			continue
+		}
+		js.userCache[jfID] = user
+		js.jsToJfID[jellyseerrID] = jfID
+		delete(js.invalidatedUsers, jellyseerrID)
+
+	}
+	return nil
+}
+
 func (js *Jellyseerr) GetPermissions(jfID string) (Permissions, error) {
 	data := permissionsDTO{Permissions: -1}
 	u, err := js.getUser(jfID)
@@ -295,6 +338,7 @@ func (js *Jellyseerr) SetPermissions(jfID string, perm Permissions) error {
 	}
 	u.Permissions = perm
 	js.userCache[jfID] = u
+	js.jsToJfID[u.ID] = jfID
 	return nil
 }
 
@@ -310,6 +354,7 @@ func (js *Jellyseerr) ApplyTemplateToUser(jfID string, tmpl UserTemplate) error
 	}
 	u.UserTemplate = tmpl
 	js.userCache[jfID] = u
+	js.jsToJfID[u.ID] = jfID
 	return nil
 }
 
@@ -326,8 +371,7 @@ func (js *Jellyseerr) ModifyUser(jfID string, conf map[UserField]any) error {
 	if err != nil {
 		return err
 	}
-	// Lazily just invalidate the cache.
-	js.cacheExpiry = time.Now()
+	js.invalidatedUsers[u.ID] = true
 	return nil
 }
 
@@ -413,12 +457,19 @@ func (js *Jellyseerr) ModifyMainUserSettings(jfID string, conf MainUserSettings)
 	if err != nil {
 		return err
 	}
+	return js.ModifyMainUserSettingsByID(u.ID, conf)
+}
 
-	_, _, err = js.post(fmt.Sprintf(js.server+"/user/%d/settings/main", u.ID), conf, false)
+func (js *Jellyseerr) ModifyMainUserSettingsByID(jellyseerrID int64, conf MainUserSettings) error {
+	_, _, err := js.post(fmt.Sprintf(js.server+"/user/%d/settings/main", jellyseerrID), conf, false)
 	if err != nil {
 		return err
 	}
-	// Lazily just invalidate the cache.
-	js.cacheExpiry = time.Now()
+	js.invalidatedUsers[jellyseerrID] = true
 	return nil
 }
+
+func (js *Jellyseerr) ReloadCache() error {
+	js.cacheExpiry = time.Now()
+	return js.getUsers()
+}

main.go

@@ -516,11 +516,16 @@ func start(asDaemon, firstCall bool) {
 		go app.userDaemon.run()
 		defer app.userDaemon.Shutdown()
 
-		if app.config.Section("jellyseerr").Key("enabled").MustBool(false) && app.config.Section("jellyseerr").Key("import_existing").MustBool(false) {
+		if app.config.Section("jellyseerr").Key("enabled").MustBool(false) {
+			// import_existing_users setting is deprecated, now it'll run when jellyseerr is enabled, or when triggered manually.
 			// jellyseerrDaemon = newJellyseerrDaemon(time.Duration(30*time.Second), app)
-			app.jellyseerrDaemon = newJellyseerrDaemon(time.Duration(10*time.Minute), app)
-			go app.jellyseerrDaemon.run()
-			defer app.jellyseerrDaemon.Shutdown()
+			app.jellyseerrDaemon = newJellyseerrDaemon(time.Duration(24*time.Hour), app)
+			if app.jellyseerrDaemon != nil {
+				go app.jellyseerrDaemon.run()
+				// Run on startup
+				go app.jellyseerrDaemon.Trigger()
+				defer app.jellyseerrDaemon.Shutdown()
+			}
 		}
 
 		if app.config.Section("password_resets").Key("enabled").MustBool(false) && serverType == mediabrowser.JellyfinServer {

migrations.go

@@ -22,6 +22,7 @@ func runMigrations(app *appContext) {
 	// migrateHyphens(app)
 	migrateToBadger(app)
 	intialiseCustomContent(app)
+	migrateJellyseerrImportDaemon(app)
 }
 
 // Migrate pre-0.2.0 user templates to profiles
@@ -505,3 +506,11 @@ func migrateExternalURL(app *appContext) {
 		return
 	}
 }
+
+// Migrate from use of "Import Existing Users" Jellyseerr import daemon to one-time run when enabling the feature.
+func migrateJellyseerrImportDaemon(app *appContext) {
+	// When Jellyseerr is disabled, set this flag to false so that an initial sync happens the next time it is enabled.
+	if !(app.config.Section("jellyseerr").Key("enabled").MustBool(false)) {
+		app.storage.db.Upsert("jellyseerr_inital_sync_status", JellyseerrInitialSyncStatus{false})
+	}
+}

ts/modules/accounts.ts

@@ -2190,6 +2190,12 @@ export class accountsList extends PaginatedList {
         this.focusAccount(userID);
     }
 
+}
+
+// An alternate view showing accounts in sub-lists grouped by group/label.
+export class groupedAccountsList {
+
+    
 }
 
 export const accountURLEvent = (id: string) => { return new CustomEvent(accountsList._accountURLEvent, {"detail": id}) };