BreizhHardware/jfa-go
1
0
Fork 0
mirror of https://github.com/hrfee/jfa-go.git synced 2026-01-18 16:47:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

accounts: infinite scroll for performance

Browse Source 
Found out the bottleneck when ~2000 or more elements are loaded isn't
the search or sort or anything, but the DOM. An infinite scroll
implementation is added, where elements are added to the DOM as you
scroll. May still be a little buggy, and can't yet cope with screen
resizes. Also, the "shown" indicator is broken.
This commit is contained in:
Harvey Tindall
2025-05-22 21:08:25 +01:00
parent 3299398806
commit d09ee59a1a
7 changed files with 216 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
config/config-base.yaml
View File

@@ -65,6 +65,20 @@ sections:
type: number
value: 30
description: Timeout of user cache in minutes. Set to 0 to disable.
- setting: web_cache_async_timeout
name: User search cache asynchronous timeout (minutes)
requires_restart: true
advanced: true
type: number
value: 1
description: "Synchronise after cache is this old, but don't wait for it: The accounts tab will load quickly but show old results until the next request."
- setting: web_cache_sync_timeout
name: User search cache synchronous timeout (minutes)
requires_restart: true
advanced: true
type: number
value: 10
description: "Synchronise after cache is this old, and wait for it: The accounts tab may take a little longer to load while it does."
- setting: type
name: Server type
requires_restart: true

2
ts/admin.ts
View File

@@ -192,7 +192,7 @@ login.onLogin = () => {
window.updater = new Updater();
// FIXME: Decide whether to autoload activity or not
reloadProfileNames();
setInterval(() => { window.invites.reload(); accounts.reload(); }, 30*1000);
setInterval(() => { window.invites.reload(); accounts.reloadIfNotInScroll(); }, 30*1000);
// Triggers pre and post funcs, even though we're already on that page
window.tabs.switch(window.tabs.current);
}

51
ts/modules/accounts.ts
View File

@@ -1,11 +1,13 @@
import { _get, _post, _delete, toggleLoader, addLoader, removeLoader, toDateString, insertText, toClipboard } from "../modules/common.js";
import { templateEmail } from "../modules/settings.js";
import { _get, _post, _delete, toggleLoader, addLoader, removeLoader, toDateString, insertText, toClipboard } from "../modules/common"
import { templateEmail } from "../modules/settings"
import { Marked } from "@ts-stack/markdown";
import { stripMarkdown } from "../modules/stripmd.js";
import { DiscordUser, newDiscordSearch } from "../modules/discord.js";
import { Search, SearchConfiguration, QueryType, SearchableItem, SearchableItems } from "../modules/search.js";
import { HiddenInputField } from "./ui.js";
import { PaginatedList } from "./list.js";
import { stripMarkdown } from "../modules/stripmd"
import { DiscordUser, newDiscordSearch } from "../modules/discord"
import { SearchConfiguration, QueryType, SearchableItem, SearchableItemDataAttribute } from "../modules/search"
import { HiddenInputField } from "./ui"
import { PaginatedList } from "./list"
// FIXME: Find and define a threshold after which searches are no longer performed on input (or just in general by the browser).
declare var window: GlobalWindow;
@@ -780,7 +782,10 @@ class user implements User, SearchableItem {
});
get id() { return this._id; }
set id(v: string) { this._id = v; }
set id(v: string) {
this._id = v;
this._row.setAttribute(SearchableItemDataAttribute, v);
}
update = (user: User) => {
@@ -821,10 +826,8 @@ interface UsersDTO extends paginatedDTO {
users: User[];
}
interface UsersReqDTO extends PaginatedReqDTO {};
export class accountsList extends PaginatedList {
private _table = document.getElementById("accounts-list") as HTMLTableSectionElement;
protected _container = document.getElementById("accounts-list") as HTMLTableSectionElement;
private _addUserButton = document.getElementById("accounts-add-user") as HTMLSpanElement;
private _announceButton = document.getElementById("accounts-announce") as HTMLSpanElement;
@@ -973,6 +976,9 @@ export class accountsList extends PaginatedList {
});
this._populateNumbers();
// FIXME: Remove!
(window as any).s = this;
let searchConfig: SearchConfiguration = {
filterArea: this._c.filterArea,
sortingByButton: this._sortingByButton,
@@ -1253,7 +1259,7 @@ export class accountsList extends PaginatedList {
set selectAll(state: boolean) {
let count = 0;
for (let id in this.users) {
if (this._table.contains(this.users[id].asElement())) { // Only select visible elements
if (this._container.contains(this.users[id].asElement())) { // Only select visible elements
this.users[id].selected = state;
count++;
}
@@ -1268,7 +1274,7 @@ export class accountsList extends PaginatedList {
for (let id of this._search.ordering) {
if (!(inRange || id == startID)) continue;
inRange = true;
if (!(this._table.contains(this.users[id].asElement()))) continue;
if (!(this._container.contains(this.users[id].asElement()))) continue;
this.users[id].selected = true;
if (id == endID) return;
}
@@ -1300,7 +1306,7 @@ export class accountsList extends PaginatedList {
} else {
let visibleCount = 0;
for (let id in this.users) {
if (this._table.contains(this.users[id].asElement())) {
if (this._container.contains(this.users[id].asElement())) {
visibleCount++;
}
}
@@ -1403,7 +1409,7 @@ export class accountsList extends PaginatedList {
private _collectUsers = (): string[] => {
let list: string[] = [];
for (let id in this.users) {
if (this._table.contains(this.users[id].asElement()) && this.users[id].selected) { list.push(id); }
if (this._container.contains(this.users[id].asElement()) && this.users[id].selected) { list.push(id); }
}
return list;
}
@@ -2065,20 +2071,7 @@ export class accountsList extends PaginatedList {
this._displayExpiryDate();
window.modals.extendExpiry.show();
}
setVisibility = (users: string[], visible: boolean) => {
console.log(`setting ${users.length} users as visible: ${visible}`);
this._table.textContent = "";
for (let id of this._search.ordering) {
if (visible && users.indexOf(id) != -1) {
this._table.appendChild(this.users[id].asElement());
} else if (!visible && users.indexOf(id) == -1) {
this._table.appendChild(this.users[id].asElement());
}
}
}
private _populateAddUserProfiles = () => {
this._addUserProfile.textContent = "";
let innerHTML = `<option value="none">${window.lang.strings("inviteNoProfile")}</option>`;

23
ts/modules/activity.ts
View File

@@ -1,5 +1,5 @@
import { _get, _post, _delete, toDateString } from "../modules/common.js";
import { SearchConfiguration, QueryType, SearchableItem, SearchableItems } from "../modules/search.js";
import { SearchConfiguration, QueryType, SearchableItem, SearchableItems, SearchableItemDataAttribute } from "../modules/search.js";
import { accountURLEvent } from "../modules/accounts.js";
import { inviteURLEvent } from "../modules/invites.js";
import { PaginatedList } from "./list.js";
@@ -353,7 +353,10 @@ export class Activity implements activity, SearchableItem {
}
get id(): string { return this._act.id; }
set id(v: string) { this._act.id = v; }
set id(v: string) {
this._act.id = v;
this._card.setAttribute(SearchableItemDataAttribute, v);
}
get user_id(): string { return this._act.user_id; }
set user_id(v: string) { this._act.user_id = v; }
@@ -379,6 +382,7 @@ export class Activity implements activity, SearchableItem {
this._card = document.createElement("div");
this._card.classList.add("card", "@low", "my-2");
this._card.innerHTML = `
<div class="flex flex-col md:flex-row justify-between mb-2">
<span class="heading truncate flex-initial md:text-2xl text-xl activity-title"></span>
@@ -476,7 +480,7 @@ interface ActivitiesDTO extends paginatedDTO {
}
export class activityList extends PaginatedList {
protected _activityList: HTMLElement;
protected _container: HTMLElement;
// protected _sortingByButton = document.getElementById("activity-sort-by-field") as HTMLButtonElement;
protected _sortDirection = document.getElementById("activity-sort-direction") as HTMLButtonElement;
@@ -525,7 +529,7 @@ export class activityList extends PaginatedList {
}
});
this._activityList = document.getElementById("activity-card-list")
this._container = document.getElementById("activity-card-list")
document.addEventListener("activity-reload", this.reload);
let searchConfig: SearchConfiguration = {
@@ -553,17 +557,6 @@ export class activityList extends PaginatedList {
this._sortDirection.addEventListener("click", () => this.ascending = !this.ascending);
}
setVisibility = (activities: string[], visible: boolean) => {
this._activityList.textContent = ``;
for (let id of this._search.ordering) {
if (visible && activities.indexOf(id) != -1) {
this._activityList.appendChild(this.activities[id].asElement());
} else if (!visible && activities.indexOf(id) == -1) {
this._activityList.appendChild(this.activities[id].asElement());
}
}
}
reload = () => {
this._reload();
}

17
ts/modules/common.ts
View File

@@ -306,3 +306,20 @@ export function unicodeB64Encode(s: string): string {
const bin = String.fromCodePoint(...encoded);
return btoa(bin);
}
// Only allow running a function every n milliseconds.
// Source: Clément Prévost at https://stackoverflow.com/questions/27078285/simple-throttle-in-javascript
// function foo<T>(bar: T): T {
export function throttle (callback: () => void, limitMilliseconds: number): () => void {
var waiting = false; // Initially, we're not waiting
return function () { // We return a throttled function
if (!waiting) { // If we're not waiting
callback.apply(this, arguments); // Execute users function
waiting = true; // Prevent future invocations
setTimeout(function () { // After a period of time
waiting = false; // And allow future invocations
}, limitMilliseconds);
}
}
}

158
ts/modules/list.ts
View File

@@ -1,5 +1,5 @@
import { _get, _post, addLoader, removeLoader } from "./common";
import { Search, SearchConfiguration, SearchableItems } from "./search";
import { _get, _post, addLoader, removeLoader, throttle } from "./common";
import { Search, SearchConfiguration } from "./search";
declare var window: GlobalWindow;
@@ -93,7 +93,22 @@ export interface PaginatedListConfig {
export abstract class PaginatedList {
protected _c: PaginatedListConfig;
// Container to append items to.
protected _container: HTMLElement;
// List of visible IDs (i.e. those set with setVisibility).
protected _visible: string[];
protected _scroll = {
rowHeight: 0,
screenHeight: 0,
// Render this many screen's worth of content below the viewport.
renderNExtraScreensWorth: 3,
rowsOnPage: 0,
rendered: 0,
initialRenderCount: 0,
scrollLoading: false
};
protected _search: Search;
protected _counter: RecordCounter;
@@ -148,7 +163,6 @@ export abstract class PaginatedList {
this.loadMore(() => removeLoader(this._keepSearchingButton, true));
}; */
// Since this.reload doesn't exist, we need an arrow function to wrap it.
// FIXME: Make sure it works though!
this._c.refreshButton.onclick = () => this.reload();
}
@@ -198,7 +212,80 @@ export abstract class PaginatedList {
};
// Sets the elements with "name"s in "elements" as visible or not.
public abstract setVisibility: (elements: string[], visible: boolean) => void;
setVisibilityNaive = (elements: string[], visible: boolean) => {
let timer = this._search.timeSearches ? performance.now() : null;
if (visible) this._visible = elements;
else this._visible = this._search.ordering.filter(v => !elements.includes(v));
const frag = document.createDocumentFragment()
for (let i = 0; i < this._visible.length; i++) {
frag.appendChild(this._search.items[this._visible[i]].asElement())
}
this._container.replaceChildren(frag);
if (this._search.timeSearches) {
const totalTime = performance.now() - timer;
console.log(`setVisibility took ${totalTime}ms`);
}
}
// FIXME: Call on window resize/zoom
// FIXME: On reload, load enough pages to fill required space.
// FIXME: Might have broken _counter.shown!
// Sets the elements with "name"s in "elements" as visible or not.
// appendedItems==true implies "elements" is the previously rendered elements plus some new ones on the end. Knowing this means the page's infinite scroll doesn't have to be reset.
setVisibility = (elements: string[], visible: boolean, appendedItems: boolean = false) => {
let timer = this._search.timeSearches ? performance.now() : null;
if (visible) this._visible = elements;
else this._visible = this._search.ordering.filter(v => !elements.includes(v));
if (this._visible.length == 0) return;
this._scroll.screenHeight = Math.max(
document.documentElement.clientHeight,
window.innerHeight || 0
);
if (!appendedItems) {
// Wipe old elements and render 1 new one, so we can take the element height.
this._container.replaceChildren(this._search.items[this._visible[0]].asElement())
}
this.computeScrollInfo();
let baseIndex = 1;
if (appendedItems) {
baseIndex = this._scroll.rendered;
}
const frag = document.createDocumentFragment()
for (let i = baseIndex; i < this._scroll.initialRenderCount; i++) {
frag.appendChild(this._search.items[this._visible[i]].asElement())
}
this._scroll.rendered = Math.max(baseIndex, this._scroll.initialRenderCount);
// appendChild over replaceChildren because there's already elements on the DOM
this._container.appendChild(frag);
if (this._search.timeSearches) {
const totalTime = performance.now() - timer;
console.log(`setVisibility took ${totalTime}ms`);
}
}
// Computes required scroll info, requiring one on-DOM item. Should be computed on page resize and this._visible change.
computeScrollInfo = () => {
this._scroll.rowHeight = this._search.items[this._visible[0]].asElement().offsetHeight;
// We want to have _scroll.renderNScreensWorth*_scroll.screenHeight or more elements rendered always.
this._scroll.rowsOnPage = Math.floor(this._scroll.screenHeight / this._scroll.rowHeight);
// Initial render of min(_visible.length, max(rowsOnPage*renderNExtraScreensWorth, itemsPerPage)), skipping 1 as we already did it.
this._scroll.initialRenderCount = Math.min(this._visible.length, Math.max((this._scroll.renderNExtraScreensWorth+1)*this._scroll.rowsOnPage, this._c.itemsPerPage));
}
// returns the item index to render up to for the given scroll position.
// might return a value greater than this._visible.length, indicating a need for a page load.
maximumItemsToRender = (scrollY: number): number => {
const bottomScroll = scrollY + ((this._scroll.renderNExtraScreensWorth+1)*this._scroll.screenHeight);
const bottomIdx = Math.floor(bottomScroll / this._scroll.rowsOnPage);
return bottomIdx;
}
// Removes all elements, and reloads the first page.
// FIXME: Share more code between reload and loadMore, and go over the logic, it's messy.
@@ -309,26 +396,65 @@ export abstract class PaginatedList {
}
this._search.onSearchBoxChange(true, loadAll);
} else {
this.setVisibility(this._search.ordering, true);
// Since results come to us ordered already, we can assume "ordering"
// will be identical to pre-page-load but with extra elements at the end,
// allowing infinite scroll to continue
this.setVisibility(this._search.ordering, true, true);
this._search.setNotFoundPanelVisibility(false);
}
if (this._c.pageLoadCallback) this._c.pageLoadCallback(req);
if (callback) callback(req);
}, true)
}
// Should be assigned to window.onscroll whenever the list is in view.
detectScroll = () => {
if (!this._hasLoaded || this.lastPage) return;
// console.log(window.innerHeight + document.documentElement.scrollTop, document.scrollingElement.scrollHeight);
if (Math.abs(window.innerHeight + document.documentElement.scrollTop - document.scrollingElement.scrollHeight) < 50) {
// window.notifications.customSuccess("scroll", "Reached bottom.");
// Wait .5s between loads
if (this._lastLoad + 500 > Date.now()) return;
this.loadMore(null, false);
loadNItems = (n: number) => {
const cb = () => {
if (this._counter.loaded > n) return;
this.loadMore(cb, false);
}
cb();
}
// As reloading can disrupt long-scrolling, this function will only do it if you're at the top of the page, essentially.
public reloadIfNotInScroll = () => {
if (this.maximumItemsToRender(window.scrollY) < this._scroll.initialRenderCount) {
return this.reload();
}
}
_detectScroll = () => {
if (!this._hasLoaded || this._scroll.scrollLoading) return;
if (this._visible.length == 0) return;
const endIdx = this.maximumItemsToRender(window.scrollY);
// If you've scrolled back up, do nothing
if (endIdx <= this._scroll.rendered) return;
const realEndIdx = Math.min(endIdx, this._visible.length);
const frag = document.createDocumentFragment();
for (let i = this._scroll.rendered; i < realEndIdx; i++) {
frag.appendChild(this._search.items[this._visible[i]].asElement());
}
this._scroll.rendered = realEndIdx;
this._container.appendChild(frag);
if (endIdx >= this._visible.length) {
if (this.lastPage || this._lastLoad + 500 > Date.now()) return;
this._scroll.scrollLoading = true;
const cb = () => {
if (this._visible.length < endIdx && !this.lastPage) {
this.loadMore(cb, false)
return;
}
this._scroll.scrollLoading = false;
this._detectScroll();
};
cb();
return;
}
}
// Should be assigned to window.onscroll whenever the list is in view.
detectScroll = throttle(this._detectScroll, 200);
}

18
ts/modules/search.ts
View File

@@ -269,8 +269,12 @@ export class DateQuery extends Query {
export interface SearchableItem {
matchesSearch: (query: string) => boolean;
// FIXME: SearchableItem should really be ListItem or something, this isn't for search!
asElement: () => HTMLElement;
}
export const SearchableItemDataAttribute = "data-search-item";
export type SearchableItems = { [id: string]: SearchableItem };
export class Search {
@@ -294,6 +298,9 @@ export class Search {
}
}
// Intended to be set from the JS console, if true searches are timed.
timeSearches: boolean = false;
private _serverSearchButtons: HTMLElement[];
static tokenizeSearch = (query: string): string[] => {
@@ -398,6 +405,7 @@ export class Search {
// Returns a list of identifiers (used as keys in items, values in ordering).
search = (query: string): string[] => {
let timer = this.timeSearches ? performance.now() : null;
this._c.filterArea.textContent = "";
let result: string[] = [...this._ordering];
@@ -460,6 +468,10 @@ export class Search {
this._queries = queries;
this._searchTerms = searchTerms;
if (this.timeSearches) {
const totalTime = performance.now() - timer;
console.log(`Search took ${totalTime}ms`);
}
return result;
}
@@ -528,10 +540,8 @@ export class Search {
this._c.notFoundLocallyText.classList.remove("unfocused");
}
if (visible) {
console.log("showing not found panel");
this._c.notFoundPanel.classList.remove("unfocused");
} else {
console.log("hiding not found panel");
this._c.notFoundPanel.classList.add("unfocused");
}
}
@@ -644,10 +654,6 @@ export class Search {
}
constructor(c: SearchConfiguration) {
// FIXME: Remove!
if (c.search.id.includes("activity")) {
(window as any).s = this;
}
this._c = c;
this._c.search.oninput = () => {