From ebff016b5da27b994de68c8ac90d5815d1978d5c Mon Sep 17 00:00:00 2001 From: Harvey Tindall Date: Thu, 15 May 2025 19:19:51 +0100 Subject: [PATCH] accounts: add "record count", start searchable user cache RecordCounter class created from that in activityList, and put in accountsList. PageCount-type route standardized and made for /users (/users/count). Created userCache, which regularly generates the respUser list returned by /users. Added a currently dumb POST /users for searching/pagination, GET /users is now just for getting -all- users. go-getted expr, an expression language that seems like it'll be useful for evaluating local searches. We don't store this data in the badger DB, so we can't use the nice query form provided by badgerhold. --- api-activities.go | 6 +-- api-users.go | 62 +++++++++++++++++---- go.mod | 1 + go.sum | 2 + html/admin.html | 7 +-- main.go | 1 + models.go | 14 ++++- router.go | 2 + scripts/account-gen/main.go | 12 +++-- ts/modules/accounts.ts | 30 ++++++++--- ts/modules/activity.ts | 104 +++++++++++++++++++++++------------- ts/typings/d.ts | 4 ++ usercache.go | 35 ++++++++++++ 13 files changed, 213 insertions(+), 67 deletions(-) create mode 100644 usercache.go diff --git a/api-activities.go b/api-activities.go index f4b2581..5ff3b76 100644 --- a/api-activities.go +++ b/api-activities.go @@ -126,8 +126,8 @@ func (app *appContext) GetActivities(gc *gin.Context) { resp := GetActivitiesRespDTO{ Activities: make([]ActivityDTO, len(results)), - LastPage: len(results) != req.Limit, } + resp.LastPage = len(results) != req.Limit for i, act := range results { resp.Activities[i] = ActivityDTO{ @@ -173,12 +173,12 @@ func (app *appContext) DeleteActivity(gc *gin.Context) { // @Summary Returns the total number of activities stored in the database. // @Produce json -// @Success 200 {object} GetActivityCountDTO +// @Success 200 {object} PageCountDTO // @Router /activity/count [get] // @Security Bearer // @tags Activity func (app *appContext) GetActivityCount(gc *gin.Context) { - resp := GetActivityCountDTO{} + resp := PageCountDTO{} var err error resp.Count, err = app.storage.db.Count(&Activity{}, &badgerhold.Query{}) if err != nil { diff --git a/api-users.go b/api-users.go index 849bdfe..60934b1 100644 --- a/api-users.go +++ b/api-users.go @@ -337,8 +337,8 @@ func (app *appContext) PostNewUserFromInvite(nu NewUserData, req ConfirmationKey // FIXME: figure these out in a nicer way? this relies on the current ordering, // which may not be fixed. if discordEnabled { - if req.completeContactMethods[0].User != nil { - discordUser = req.completeContactMethods[0].User.(*DiscordUser) + if req.completeContactMethods[0].User != nil { + discordUser = req.completeContactMethods[0].User.(*DiscordUser) } if telegramEnabled && req.completeContactMethods[1].User != nil { telegramUser = req.completeContactMethods[1].User.(*TelegramUser) @@ -894,7 +894,25 @@ func (app *appContext) userSummary(jfUser mediabrowser.User) respUser { } -// @Summary Get a list of Jellyfin users. +// @Summary Returns the total number of Jellyfin users. +// @Produce json +// @Success 200 {object} PageCountDTO +// @Router /users/count [get] +// @Security Bearer +// @tags Activity +func (app *appContext) GetUserCount(gc *gin.Context) { + resp := PageCountDTO{} + err := app.userCache.Gen(app) + if err != nil { + app.err.Printf(lm.FailedGetUsers, lm.Jellyfin, err) + respond(500, "Couldn't get users", gc) + return + } + resp.Count = uint64(len(app.userCache.Cache)) + gc.JSON(200, resp) +} + +// @Summary Get a list of -all- Jellyfin users. // @Produce json // @Success 200 {object} getUsersDTO // @Failure 500 {object} stringResponse @@ -903,19 +921,43 @@ func (app *appContext) userSummary(jfUser mediabrowser.User) respUser { // @tags Users func (app *appContext) GetUsers(gc *gin.Context) { var resp getUsersDTO - users, err := app.jf.GetUsers(false) - resp.UserList = make([]respUser, len(users)) + // We're sending all users, so this is always true + resp.LastPage = true + err := app.userCache.Gen(app) if err != nil { app.err.Printf(lm.FailedGetUsers, lm.Jellyfin, err) respond(500, "Couldn't get users", gc) return } - i := 0 - for _, jfUser := range users { - user := app.userSummary(jfUser) - resp.UserList[i] = user - i++ + resp.UserList = app.userCache.Cache + gc.JSON(200, resp) +} + +// @Summary Get a paginated, searchable list of Jellyfin users. +// @Produce json +// @Param getUsersReqDTO body getUsersReqDTO true "search / pagination parameters" +// @Success 200 {object} getUsersDTO +// @Failure 500 {object} stringResponse +// @Router /users [post] +// @Security Bearer +// @tags Users +func (app *appContext) SearchUsers(gc *gin.Context) { + req := getUsersReqDTO{} + gc.BindJSON(&req) + + // FIXME: Figure out how to search, sort and paginate []mediabrowser.User! + // Expr! + + var resp getUsersDTO + // We're sending all users, so this is always true + resp.LastPage = true + err := app.userCache.Gen(app) + if err != nil { + app.err.Printf(lm.FailedGetUsers, lm.Jellyfin, err) + respond(500, "Couldn't get users", gc) + return } + resp.UserList = app.userCache.Cache gc.JSON(200, resp) } diff --git a/go.mod b/go.mod index 6b17a34..eb0fcb0 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/dgraph-io/ristretto v1.0.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/expr-lang/expr v1.17.3 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 // indirect github.com/getlantern/errors v1.0.4 // indirect diff --git a/go.sum b/go.sum index c9c85b6..a12698d 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/expr-lang/expr v1.17.3 h1:myeTTuDFz7k6eFe/JPlep/UsiIjVhG61FMHFu63U7j0= +github.com/expr-lang/expr v1.17.3/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= diff --git a/html/admin.html b/html/admin.html index 25630bd..72953a0 100644 --- a/html/admin.html +++ b/html/admin.html @@ -738,6 +738,7 @@ +
{{ .strings.actions }}
{{ .quantityStrings.addUser.Singular }} @@ -838,11 +839,7 @@
-
- - - -
+
diff --git a/main.go b/main.go index 7d1de5a..cb6d2c4 100644 --- a/main.go +++ b/main.go @@ -134,6 +134,7 @@ type appContext struct { pwrCaptchas map[string]Captcha ConfirmationKeys map[string]map[string]ConfirmationKey // Map of invite code to jwt to request confirmationKeysLock sync.Mutex + userCache UserCache } func generateSecret(length int) (string, error) { diff --git a/models.go b/models.go index 686effd..360a2e2 100644 --- a/models.go +++ b/models.go @@ -164,8 +164,18 @@ type respUser struct { ReferralsEnabled bool `json:"referrals_enabled"` } +type PaginatedDTO struct { + LastPage bool `json:"last_page"` +} + +type getUsersReqDTO struct { + Limit int `json:"limit"` + Page int `json:"page"` // zero-indexed +} + type getUsersDTO struct { UserList []respUser `json:"users"` + LastPage bool `json:"last_page"` } type ombiUser struct { @@ -437,11 +447,11 @@ type GetActivitiesDTO struct { } type GetActivitiesRespDTO struct { + PaginatedDTO Activities []ActivityDTO `json:"activities"` - LastPage bool `json:"last_page"` } -type GetActivityCountDTO struct { +type PageCountDTO struct { Count uint64 `json:"count"` } diff --git a/router.go b/router.go index 9ab3530..4fa3952 100644 --- a/router.go +++ b/router.go @@ -183,6 +183,8 @@ func (app *appContext) loadRoutes(router *gin.Engine) { router.POST(p+"/logout", app.Logout) api.DELETE(p+"/users", app.DeleteUsers) api.GET(p+"/users", app.GetUsers) + api.GET(p+"/users/count", app.GetUserCount) + api.POST(p+"/users", app.SearchUsers) api.POST(p+"/user", app.NewUserFromAdmin) api.POST(p+"/users/extend", app.ExtendExpiry) api.DELETE(p+"/users/:id/expiry", app.RemoveExpiry) diff --git a/scripts/account-gen/main.go b/scripts/account-gen/main.go index bc1c528..47faa6f 100644 --- a/scripts/account-gen/main.go +++ b/scripts/account-gen/main.go @@ -15,11 +15,11 @@ import ( var ( names = []string{"Aaron", "Agnes", "Bridget", "Brandon", "Dolly", "Drake", "Elizabeth", "Erika", "Geoff", "Graham", "Haley", "Halsey", "Josie", "John", "Kayleigh", "Luka", "Melissa", "Nasreen", "Paul", "Ross", "Sam", "Talib", "Veronika", "Zaynab"} + COUNT = 3000 ) const ( PASSWORD = "test" - COUNT = 10 ) func main() { @@ -57,6 +57,12 @@ func main() { password = strings.TrimSuffix(password, "\n") } + if countEnv := os.Getenv("COUNT"); countEnv != "" { + COUNT, _ = strconv.Atoi(countEnv) + } + + fmt.Printf("Will generate %d users\n", COUNT) + jf, err := mediabrowser.NewServer( mediabrowser.JellyfinServer, server, @@ -99,7 +105,7 @@ func main() { user, status, err := jf.NewUser(name, PASSWORD) if (status != 200 && status != 201 && status != 204) || err != nil { - log.Fatalf("Failed to create user \"%s\" (%d): %+v\n", name, status, err) + log.Fatalf("Acc no %d: Failed to create user \"%s\" (%d): %+v\n", i, name, status, err) } if rand.Intn(100) > 65 { @@ -112,7 +118,7 @@ func main() { status, err = jf.SetPolicy(user.ID, user.Policy) if (status != 200 && status != 201 && status != 204) || err != nil { - log.Fatalf("Failed to set policy for user \"%s\" (%d): %+v\n", name, status, err) + log.Fatalf("Acc no %d: Failed to set policy for user \"%s\" (%d): %+v\n", i, name, status, err) } if rand.Intn(100) > 20 { diff --git a/ts/modules/accounts.ts b/ts/modules/accounts.ts index c338bfc..2bc902d 100644 --- a/ts/modules/accounts.ts +++ b/ts/modules/accounts.ts @@ -5,6 +5,7 @@ import { stripMarkdown } from "../modules/stripmd.js"; import { DiscordUser, newDiscordSearch } from "../modules/discord.js"; import { Search, SearchConfiguration, QueryType, SearchableItem } from "../modules/search.js"; import { HiddenInputField } from "./ui.js"; +import { RecordCounter } from "./activity.js"; declare var window: GlobalWindow; @@ -702,7 +703,11 @@ class user implements User, SearchableItem { } this._row.remove(); } -} +} + +interface UsersDTO extends paginatedDTO { + users: User[]; +} export class accountsList { private _table = document.getElementById("accounts-list") as HTMLTableSectionElement; @@ -771,6 +776,8 @@ export class accountsList { private _filterArea = document.getElementById("accounts-filter-area"); private _searchOptionsHeader = document.getElementById("accounts-search-options-header"); + private _counter: RecordCounter; + // Whether the "Extend expiry" is extending or setting an expiry. private _settingExpiry = false; @@ -1779,6 +1786,9 @@ export class accountsList { constructor() { this._populateNumbers(); + + this._counter = new RecordCounter(document.getElementById("accounts-record-counter")); + this._users = {}; this._selectAll.checked = false; this._selectAll.onchange = () => { @@ -2035,12 +2045,16 @@ export class accountsList { } reload = (callback?: () => void) => { + this._counter.reset() + this._counter.getTotal("/users/count"); + _get("/users", null, (req: XMLHttpRequest) => { if (req.readyState == 4 && req.status == 200) { + let resp = req.response as UsersDTO; // same method as inviteList.reload() let accountsOnDOM: { [id: string]: boolean } = {}; for (let id in this._users) { accountsOnDOM[id] = true; } - for (let u of (req.response["users"] as User[])) { + for (let u of resp.users) { if (u.id in this._users) { this._users[u.id].update(u); delete accountsOnDOM[u.id]; @@ -2055,10 +2069,10 @@ export class accountsList { // console.log("reload, so sorting by", this._activeSortColumn); this._ordering = this._columns[this._activeSortColumn].sort(this._users); this._search.ordering = this._ordering; - if (!(this._search.inSearch)) { - this.setVisibility(this._ordering, true); - this._notFoundPanel.classList.add("unfocused"); - } else { + + this._counter.loaded = this._ordering.length; + + if (this._search.inSearch) { const results = this._search.search(this._searchBox.value); if (results.length == 0) { this._notFoundPanel.classList.remove("unfocused"); @@ -2066,6 +2080,10 @@ export class accountsList { this._notFoundPanel.classList.add("unfocused"); } this.setVisibility(results, true); + } else { + this._counter.shown = this._counter.loaded; + this.setVisibility(this._ordering, true); + this._notFoundPanel.classList.add("unfocused"); } this._checkCheckCount(); diff --git a/ts/modules/activity.ts b/ts/modules/activity.ts index 135d50b..59ed865 100644 --- a/ts/modules/activity.ts +++ b/ts/modules/activity.ts @@ -346,9 +346,64 @@ export class Activity implements activity, SearchableItem { asElement = () => { return this._card; }; } -interface ActivitiesDTO { +export class RecordCounter { + private _container: HTMLElement; + private _totalRecords: HTMLElement; + private _loadedRecords: HTMLElement; + private _shownRecords: HTMLElement; + private _total: number; + private _loaded: number; + private _shown: number; + constructor(container: HTMLElement) { + this._container = container; + this._container.innerHTML = ` + + + + `; + this._totalRecords = document.getElementsByClassName("records-total")[0] as HTMLElement; + this._loadedRecords = document.getElementsByClassName("records-loaded")[0] as HTMLElement; + this._shownRecords = document.getElementsByClassName("records-shown")[0] as HTMLElement; + this.total = 0; + this.loaded = 0; + this.shown = 0; + } + + reset() { + this.total = 0; + this.loaded = 0; + this.shown = 0; + } + + // Sets the total using a PageCountDTO-returning API endpoint. + getTotal(endpoint: string) { + _get(endpoint, null, (req: XMLHttpRequest) => { + if (req.readyState != 4 || req.status != 200) return; + this.total = req.response["count"] as number; + }); + } + + get total(): number { return this._total; } + set total(v: number) { + this._total = v; + this._totalRecords.textContent = window.lang.var("strings", "totalRecords", `${v}`); + } + + get loaded(): number { return this._loaded; } + set loaded(v: number) { + this._loaded = v; + this._loadedRecords.textContent = window.lang.var("strings", "loadedRecords", `${v}`); + } + + get shown(): number { return this._shown; } + set shown(v: number) { + this._shown = v; + this._shownRecords.textContent = window.lang.var("strings", "shownRecords", `${v}`); + } +} + +interface ActivitiesDTO extends paginatedDTO { activities: activity[]; - last_page: boolean; } export class activityList { @@ -368,31 +423,7 @@ export class activityList { private _keepSearchingDescription = document.getElementById("activity-keep-searching-description"); private _keepSearchingButton = document.getElementById("activity-keep-searching"); - private _totalRecords = document.getElementById("activity-total-records"); - private _loadedRecords = document.getElementById("activity-loaded-records"); - private _shownRecords = document.getElementById("activity-shown-records"); - - private _total: number; - private _loaded: number; - private _shown: number; - - get total(): number { return this._total; } - set total(v: number) { - this._total = v; - this._totalRecords.textContent = window.lang.var("strings", "totalRecords", `${v}`); - } - - get loaded(): number { return this._loaded; } - set loaded(v: number) { - this._loaded = v; - this._loadedRecords.textContent = window.lang.var("strings", "loadedRecords", `${v}`); - } - - get shown(): number { return this._shown; } - set shown(v: number) { - this._shown = v; - this._shownRecords.textContent = window.lang.var("strings", "shownRecords", `${v}`); - } + private _counter: RecordCounter; private _search: Search; private _ascending: boolean; @@ -421,9 +452,8 @@ export class activityList { this._loadAllButton.classList.remove("unfocused"); this._loadAllButton.disabled = false; - this.total = 0; - this.loaded = 0; - this.shown = 0; + this._counter.reset(); + this._counter.getTotal("/activity/count"); // this._page = 0; let limit = 10; @@ -438,10 +468,6 @@ export class activityList { "ascending": this.ascending } - _get("/activity/count", null, (req: XMLHttpRequest) => { - if (req.readyState != 4 || req.status != 200) return; - this.total = req.response["count"] as number; - }); _post("/activity", send, (req: XMLHttpRequest) => { if (req.readyState != 4) return; @@ -467,13 +493,13 @@ export class activityList { this._search.items = this._activities; this._search.ordering = this._ordering; - this.loaded = this._ordering.length; + this._counter.loaded = this._ordering.length; if (this._search.inSearch) { this._search.onSearchBoxChange(true); this._loadAllButton.classList.remove("unfocused"); } else { - this.shown = this.loaded; + this._counter.shown = this._counter.loaded; this.setVisibility(this._ordering, true); this._loadAllButton.classList.add("unfocused"); this._notFoundPanel.classList.add("unfocused"); @@ -526,7 +552,7 @@ export class activityList { // this._search.items = this._activities; // this._search.ordering = this._ordering; - this.loaded = this._ordering.length; + this._counter.loaded = this._ordering.length; if (this._search.inSearch || loadAll) { if (this._lastPage) { @@ -699,6 +725,8 @@ export class activityList { this._activityList = document.getElementById("activity-card-list"); document.addEventListener("activity-reload", this.reload); + this._counter = new RecordCounter(document.getElementById("activity-record-counter")); + let conf: SearchConfiguration = { filterArea: this._filterArea, sortingByButton: this._sortingByButton, @@ -711,7 +739,7 @@ export class activityList { filterList: document.getElementById("activity-filter-list"), // notFoundCallback: this._notFoundCallback, onSearchCallback: (visibleCount: number, newItems: boolean, loadAll: boolean) => { - this.shown = visibleCount; + this._counter.shown = visibleCount; if (this._search.inSearch && !this._lastPage) this._loadAllButton.classList.remove("unfocused"); else this._loadAllButton.classList.add("unfocused"); diff --git a/ts/typings/d.ts b/ts/typings/d.ts index b19a1c7..a4f7f80 100644 --- a/ts/typings/d.ts +++ b/ts/typings/d.ts @@ -155,5 +155,9 @@ interface inviteList { // submitter: HTMLInputElement; // } +interface paginatedDTO { + last_page: boolean; +} + declare var config: Object; declare var modifiedConfig: Object; diff --git a/usercache.go b/usercache.go new file mode 100644 index 0000000..68af907 --- /dev/null +++ b/usercache.go @@ -0,0 +1,35 @@ +package main + +import ( + "sync" + "time" +) + +const ( + // FIXME: Follow mediabrowser, or make tuneable, or both + WEB_USER_CACHE_SYNC = 30 * time.Second +) + +type UserCache struct { + Cache []respUser + LastSync time.Time + Lock sync.Mutex +} + +func (c *UserCache) Gen(app *appContext) error { + if !time.Now().After(c.LastSync.Add(WEB_USER_CACHE_SYNC)) { + return nil + } + users, err := app.jf.GetUsers(false) + if err != nil { + return err + } + c.Lock.Lock() + c.Cache = make([]respUser, len(users)) + for i, jfUser := range users { + c.Cache[i] = app.userSummary(jfUser) + } + c.LastSync = time.Now() + c.Lock.Unlock() + return nil +}