Fix for library sync

Made changes to sync process Data calls are now paginated at 200 items per call Recursive search added to account for items within subfolders Reduced API calls by recursive calls on entire library vs per library/show/season
2026-03-18 21:30:35 +01:00 · 2023-06-07 11:00:04 +02:00
parent 569d1a1061
commit 2e2bab9b73
6 changed files with 184 additions and 117 deletions
--- a/backend/api.js
+++ b/backend/api.js
@@ -553,6 +553,7 @@ router.post("/getLibraries", async (req, res) => {
                raw_item_data:{},
                raw_season_data:{},
                raw_episode_data:{},
+                count_from_api:{},
              };

    /////////////////////////Get Admin
@@ -575,7 +576,15 @@ router.post("/getLibraries", async (req, res) => {
    const db_seasons=await db.query('SELECT "Id" FROM jf_library_seasons').then((res) => res.rows.map((row) => row.Id));
    const db_episodes=await db.query('SELECT "EpisodeId" FROM jf_library_episodes').then((res) => res.rows.map((row) => row.EpisodeId));

+    let count_url=`${config[0].JF_HOST}/items/counts`;
+    
+    const response_api_count = await axios_instance.get(count_url, {
+      headers: {
+        "X-MediaBrowser-Token":  config[0].JF_API_KEY ,
+      },
+    });

+    payload.count_from_api=response_api_count.data;
 //get libraries
    let url=`${config[0].JF_HOST}/Users/${adminUser[0].Id}/Items`;
    
--- a/backend/sync.js
+++ b/backend/sync.js
@@ -37,11 +37,7 @@ const {jf_users_columns,jf_users_mapping,} = require("./models/jf_users");
 /////////////////////////////////////////Functions

 function getErrorLineNumber(error) {
-  // console.log(error);
  const stackTrace = error.stack.split('\n');
-  // The stack trace will contain the file path and line number
-  // of each function call leading up to the error.
-  // We can extract the line number of the error from the stack trace.
  const errorLine = stackTrace[1].trim();
  const lineNumber = errorLine.substring(
    errorLine.lastIndexOf('\\') + 1,
@@ -79,6 +75,7 @@ class sync {
          "X-MediaBrowser-Token": this.apiKey,
        },
      });
+
      if(!response || typeof response.data !== 'object' || !Array.isArray(response.data))
      {
        res.status(503);
@@ -98,24 +95,44 @@ class sync {
    }
  }

-  async getItem(itemID,userid) {
+  async getItem(key,id,params) {
    try {

-      let url = `${this.hostUrl}/users/${userid}/Items`;
-      if (itemID !== undefined) {
-        url += `?ParentID=${itemID}`;
-      }
-      const response = await axios_instance.get(url, {
-        headers: {
-          "X-MediaBrowser-Token": this.apiKey,
-        },
-      });

-      const results = response.data.Items;
-      if (itemID === undefined) {
-        return results.filter((type) => !["boxsets","playlists"].includes(type.CollectionType));
+      let url = `${this.hostUrl}/Items?${key}=${id}`;
+      let startIndex=params && params.startIndex ? params.startIndex :0;
+      let increment=params && params.increment ? params.startIndex :200;
+      let recursive=params && params.recursive!==undefined  ? params.recursive :true;
+      let total=200;
+
+      let final_response=[];
+      while(startIndex<total && total !== undefined)
+      {
+        const response = await axios_instance.get(url, {
+          headers: {
+            "X-MediaBrowser-Token": this.apiKey,
+          },        
+          params:{
+            startIndex:startIndex,
+            recursive:recursive,
+            limit:increment
+          },
+        });
+
+        total=response.data.TotalRecordCount;
+        startIndex+=increment;
+
+         final_response=[...final_response, ...response.data.Items];
+
+      }
+
+
+      // const results = response.data.Items;
+      if (key === 'userid') {
+        return final_response.filter((type) => !["boxsets","playlists"].includes(type.CollectionType));
      } else {
-        return results.filter((item) => item.ImageTags.Primary);
+        // return final_response.filter((item) => item.ImageTags.Primary);
+        return final_response;
      }
    } catch (error) {
      console.log(error);
@@ -123,27 +140,6 @@ class sync {
    }
  }

-  async getSeasonsAndEpisodes(itemID, type) {
-    try {
-
-      let url = `${this.hostUrl}/shows/${itemID}/${type}`;
-      // if (itemID !== undefined) {
-      //   url += `?ParentID=${itemID}`;
-      // }
-      const response = await axios_instance.get(url, {
-        headers: {
-          "X-MediaBrowser-Token": this.apiKey,
-        },
-      });
-
-      return response.data.Items;
-    } catch (error) {
-      console.log(error);
-      return [];
-    }
-  }
-
-  

  async getItemInfo(itemID,userid) {
    try {
@@ -231,24 +227,11 @@ async function syncUserData(refLog)

 }

-async function syncLibraryFolders(refLog)
+async function syncLibraryFolders(refLog,data)
 {
  try
  {
-
-    const { rows } = await db.query('SELECT * FROM app_config where "ID"=1');
-    if (rows[0].JF_HOST === null || rows[0].JF_API_KEY === null) {
-      res.send({ error: "Config Details Not Found" });
-      refLog.loggedData.push({ Message: "Error: Config details not found!" });
-      refLog.result='Failed';
-      return;
-    }
-  
-    const _sync = new sync(rows[0].JF_HOST, rows[0].JF_API_KEY);
-    const admins = await _sync.getAdminUser(refLog);
-    const userid = admins[0].Id;
-    const data = await _sync.getItem(undefined,userid); //getting all root folders aka libraries
-  
+ 
    const existingIds = await db
      .query('SELECT "Id" FROM jf_libraries')
      .then((res) => res.rows.map((row) => row.Id));
@@ -295,38 +278,16 @@ async function syncLibraryFolders(refLog)
  }
  
 }
-async function syncLibraryItems(refLog)
+async function syncLibraryItems(refLog,data)
 {
  try{
-    const { rows: config } = await db.query('SELECT * FROM app_config where "ID"=1' );

-
-  if (config[0].JF_HOST === null || config[0].JF_API_KEY === null) {
-    res.send({ error: "Config Details Not Found" });
-    refLog.result='Failed';
-    return;
-  }
-
-  const _sync = new sync(config[0].JF_HOST, config[0].JF_API_KEY);
  refLog.loggedData.push({ color: "lawngreen", Message: "Syncing... 1/3" });
  refLog.loggedData.push({color: "yellow",Message: "Beginning Library Item Sync",});

-  const admins = await _sync.getAdminUser(refLog);
-  const userid = admins[0].Id;
-  const libraries = await _sync.getItem(undefined,userid);
-  const data = [];
+
  let insertMessage='';
  let deleteCounter = 0;
-  //for each item in library run get item using that id as the ParentId (This gets the children of the parent id)
-  for (let i = 0; i < libraries.length; i++) {
-    const item = libraries[i];
-    let libraryItems = await _sync.getItem(item.Id,userid);
-    const libraryItemsWithParent = libraryItems.map((items) => ({
-      ...items,
-      ...{ ParentId: item.Id },
-    }));
-    data.push(...libraryItemsWithParent);
-  }


  const existingIds = await db
@@ -377,7 +338,7 @@ async function syncLibraryItems(refLog)

 }

-async function syncShowItems(refLog)
+async function syncShowItems(refLog,data)
 {
 try{
  refLog.loggedData.push({ color: "lawngreen", Message: "Syncing... 2/3" });
@@ -391,7 +352,7 @@ async function syncShowItems(refLog)
    return;
  }

-  const _sync = new sync(config[0].JF_HOST, config[0].JF_API_KEY);
+  // const _sync = new sync(config[0].JF_HOST, config[0].JF_API_KEY);
  const { rows: shows } = await db.query(`SELECT *	FROM public.jf_library_items where "Type"='Series'`);

  let insertSeasonsCount = 0;
@@ -405,8 +366,8 @@ async function syncShowItems(refLog)

  //loop for each show
  for (const show of shows) {
-    const allSeasons = await _sync.getSeasonsAndEpisodes(show.Id,'Seasons');
-    const allEpisodes =await _sync.getSeasonsAndEpisodes(show.Id,'Episodes');
+    const allSeasons =  data.filter((item) => item.Type==='Season' && item.SeriesId===show.Id);
+    const allEpisodes =data.filter((item) => item.Type==='Episode' && item.SeriesId===show.Id);

    const existingIdsSeasons = await db.query(`SELECT *	FROM public.jf_library_seasons where "SeriesId" = '${show.Id}'`).then((res) => res.rows.map((row) => row.Id));
    let existingIdsEpisodes = [];
@@ -429,8 +390,6 @@ async function syncShowItems(refLog)
    seasonsToInsert = await allSeasons.map(jf_library_seasons_mapping);
    episodesToInsert = await allEpisodes.map(jf_library_episodes_mapping);

-    seasonsToInsert=seasonsToInsert.filter((item)=>item.Id !== undefined);
-    episodesToInsert=episodesToInsert.filter((item)=>item.Id !== undefined);
    //Bulkinsert new data not on db
    if (seasonsToInsert.length !== 0) {
      let result = await db.insertBulk("jf_library_seasons",seasonsToInsert,jf_library_seasons_columns);
@@ -712,34 +671,72 @@ async function removeOrphanedData(refLog)

 async function fullSync()
 {
-  let startTime = moment();
-  let refLog={loggedData:[],result:'Success'};
-  await syncUserData(refLog);
-  await syncLibraryFolders(refLog);
-  await syncLibraryItems(refLog);
-  await syncShowItems(refLog);
-  await syncItemInfo(refLog);
-  await removeOrphanedData(refLog);
-  const uuid = randomUUID();
-
-  let endTime = moment();
- 
-  let diffInSeconds = endTime.diff(startTime, 'seconds');
-
-  const log=
+  try
  {
-    "Id":uuid,
-    "Name":"Jellyfin Sync",
-    "Type":"Task",
-    "ExecutionType":"Automatic",
-    "Duration":diffInSeconds,
-    "TimeRun":startTime,
-    "Log":JSON.stringify(refLog.loggedData),
-    "Result":refLog.result
+    let startTime = moment();
+    let refLog={loggedData:[],result:'Success'};
+  
+    const { rows } = await db.query('SELECT * FROM app_config where "ID"=1');
+    if (rows[0].JF_HOST === null || rows[0].JF_API_KEY === null) {
+      res.send({ error: "Config Details Not Found" });
+      refLog.loggedData.push({ Message: "Error: Config details not found!" });
+      refLog.result='Failed';
+      return;
+    }
+  
+    const _sync = new sync(rows[0].JF_HOST, rows[0].JF_API_KEY);
+  
+    const admins = await _sync.getAdminUser(refLog);
+    const userid = admins[0].Id;
+    const libraries = await _sync.getItem('userid',userid,{recursive:false}); //getting all root folders aka libraries + items
+    const data=[];

-  };
-   logging.insertLog(log);
+    //for each item in library run get item using that id as the ParentId (This gets the children of the parent id)
+  for (let i = 0; i < libraries.length; i++) {
+    const item = libraries[i];
+    let libraryItems = await _sync.getItem('parentId',item.Id);
+    const libraryItemsWithParent = libraryItems.map((items) => ({
+      ...items,
+      ...{ ParentId: item.Id },
+    }));
+    data.push(...libraryItemsWithParent);
+  }
+    const library_items=data.filter((item) => ['Movie','Audio','Series'].includes(item.Type));
+    const seasons_and_episodes=data.filter((item) => ['Season','Episode'].includes(item.Type));

+    await syncUserData(refLog);
+  
+    await syncLibraryFolders(refLog,libraries);
+    await syncLibraryItems(refLog,library_items);
+    await syncShowItems(refLog,seasons_and_episodes);
+    await syncItemInfo(refLog);
+    await removeOrphanedData(refLog);
+    const uuid = randomUUID();
+  
+    let endTime = moment();
+   
+    let diffInSeconds = endTime.diff(startTime, 'seconds');
+  
+    const log=
+    {
+      "Id":uuid,
+      "Name":"Jellyfin Sync",
+      "Type":"Task",
+      "ExecutionType":"Automatic",
+      "Duration":diffInSeconds,
+      "TimeRun":startTime,
+      "Log":JSON.stringify(refLog.loggedData),
+      "Result":refLog.result
+  
+    };
+     logging.insertLog(log);
+  
+    
+  }catch(error)
+  {
+    console.log(error);
+  }
+  

 }

@@ -750,16 +747,45 @@ async function fullSync()
 router.get("/beingSync", async (req, res) => {
  socket.clearMessages();
  let refLog={loggedData:[],result:'Success'};
-
  let startTime = moment();
+
+  const { rows } = await db.query('SELECT * FROM app_config where "ID"=1');
+  if (rows[0].JF_HOST === null || rows[0].JF_API_KEY === null) {
+    res.send({ error: "Config Details Not Found" });
+    refLog.loggedData.push({ Message: "Error: Config details not found!" });
+    refLog.result='Failed';
+    return;
+  }
+
+  const _sync = new sync(rows[0].JF_HOST, rows[0].JF_API_KEY);
+
+  const admins = await _sync.getAdminUser(refLog);
+  const userid = admins[0].Id;
+  const libraries = await _sync.getItem('userid',userid,{recursive:false}); //getting all root folders aka libraries + items
+  const data=[];
+
+    //for each item in library run get item using that id as the ParentId (This gets the children of the parent id)
+  for (let i = 0; i < libraries.length; i++) {
+    const item = libraries[i];
+    let libraryItems = await _sync.getItem('parentId',item.Id);
+    const libraryItemsWithParent = libraryItems.map((items) => ({
+      ...items,
+      ...{ ParentId: item.Id },
+    }));
+    data.push(...libraryItemsWithParent);
+  }
+
+  const library_items=data.filter((item) => ['Movie','Audio','Series'].includes(item.Type));
+  const seasons_and_episodes=data.filter((item) => ['Season','Episode'].includes(item.Type));
+
  await syncUserData(refLog);
-  await syncLibraryFolders(refLog);
-  await syncLibraryItems(refLog);
-  await syncShowItems(refLog);
+
+  await syncLibraryFolders(refLog,libraries);
+  await syncLibraryItems(refLog,library_items);
+  await syncShowItems(refLog,seasons_and_episodes);
  await syncItemInfo(refLog);
  await removeOrphanedData(refLog);
  const uuid = randomUUID();
-
  let endTime = moment();
 
  let diffInSeconds = endTime.diff(startTime, 'seconds');
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,15 +1,16 @@
 {
  "name": "jfstat",
-  "version": "1.0.4.6",
+  "version": "1.0.4.8",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "jfstat",
-      "version": "1.0.4.6",
+      "version": "1.0.4.7",
      "dependencies": {
        "@emotion/react": "^11.10.6",
        "@emotion/styled": "^11.10.6",
+        "@jellyfin/sdk": "^0.8.2",
        "@mui/material": "^5.12.2",
        "@mui/x-data-grid": "^6.2.1",
        "@nivo/api": "^0.74.1",
@@ -2389,6 +2390,22 @@
        "node": ">=8"
      }
    },
+    "node_modules/@jellyfin/sdk": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/@jellyfin/sdk/-/sdk-0.8.2.tgz",
+      "integrity": "sha512-u3yfu7ODL9zN/ytQ+JomeZTkyVMuF5BR/y9tWL59tjgQ44Zf9MGBSeys3RGz60AW/OkpCP7G9MfaR3mP11pn6w==",
+      "dependencies": {
+        "compare-versions": "5.0.3"
+      },
+      "peerDependencies": {
+        "axios": "^1.3.4"
+      }
+    },
+    "node_modules/@jellyfin/sdk/node_modules/compare-versions": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-5.0.3.tgz",
+      "integrity": "sha512-4UZlZP8Z99MGEY+Ovg/uJxJuvoXuN4M6B3hKaiackiHrgzQFEe3diJi1mf1PNHbFujM7FvLrK2bpgIaImbtZ1A=="
+    },
    "node_modules/@jest/console": {
      "version": "27.5.1",
      "license": "MIT",
--- a/package.json
+++ b/package.json
@@ -5,6 +5,7 @@
  "dependencies": {
    "@emotion/react": "^11.10.6",
    "@emotion/styled": "^11.10.6",
+    "@jellyfin/sdk": "^0.8.2",
    "@mui/material": "^5.12.2",
    "@mui/x-data-grid": "^6.2.1",
    "@nivo/api": "^0.74.1",
--- a/src/pages/components/library/library-card.js
+++ b/src/pages/components/library/library-card.js
@@ -134,7 +134,7 @@ function LibraryCard(props) {
            </Row>

            <Row className="space-between-end card-row">
-              <Col className="card-label">{props.data.CollectionType==='tvshows' ? 'Series' : "Movies"}</Col>
+              <Col className="card-label">{props.data.CollectionType==='tvshows' ? 'Series' : props.data.CollectionType==='movies'? "Movies" : props.data.CollectionType==='music'? "Songs" : 'Files'}</Col>
              <Col className="text-end">{props.data.Library_Count}</Col>
            </Row>

--- a/src/pages/data-debugger.js
+++ b/src/pages/data-debugger.js
@@ -73,6 +73,8 @@ function Datadebugger() {
                          <TableCell>Database Count</TableCell>
                          <TableCell>API Count</TableCell>
                          <TableCell>Difference</TableCell>
+                          <TableCell>Counts from Jellyfin*</TableCell>
+                          <TableCell>Difference</TableCell>
                          <TableCell>Export Missing Data</TableCell>
                        </TableRow>
                      </TableHead>
@@ -83,6 +85,8 @@ function Datadebugger() {
                            <TableCell>{data ? data.existing_library_count:''}</TableCell>
                            <TableCell>{data ? data.api_library_count:''}</TableCell>
                            <TableCell>{data ? data.api_library_count-data.existing_library_count:''}</TableCell>
+                            <TableCell></TableCell>
+                            <TableCell></TableCell>
                            <TableCell>{data  &&  data.api_library_count>data.existing_library_count ?       
                              <Button onClick={()=>handleDownload(data.missing_api_library_data,'MissingLibraryData')}>
                               Download
@@ -96,6 +100,8 @@ function Datadebugger() {
                            <TableCell>{data ? data.existing_movie_count:''}</TableCell>
                            <TableCell>{data ? data.api_movie_count:''}</TableCell>
                            <TableCell>{data ? data.api_movie_count-data.existing_movie_count:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.MovieCount:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.MovieCount-data.existing_movie_count:''}</TableCell>
                            <TableCell>{data  &&  data.api_movie_count>data.existing_movie_count ?       
                              <Button onClick={()=>handleDownload(data.missing_api_movies_data,'MissingMovieData')}>
                               Download
@@ -109,6 +115,8 @@ function Datadebugger() {
                            <TableCell>{data ? data.existing_show_count:''}</TableCell>
                            <TableCell>{data ? data.api_show_count:''}</TableCell>
                            <TableCell>{data ? data.api_show_count-data.existing_show_count:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.SeriesCount:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.SeriesCount-data.existing_show_count:''}</TableCell>
                            <TableCell>{data  &&  data.api_show_count>data.existing_show_count ?       
                              <Button onClick={()=>handleDownload(data.missing_api_shows_data,'MissingShowData')}>
                               Download
@@ -122,6 +130,8 @@ function Datadebugger() {
                            <TableCell>{data ? data.existing_music_count:''}</TableCell>
                            <TableCell>{data ? data.api_music_count:''}</TableCell>
                            <TableCell>{data ? data.api_music_count-data.existing_music_count:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.SongCount:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.SongCount-data.existing_music_count:''}</TableCell>
                            <TableCell>{data  &&  data.api_music_count>data.existing_music_count ?       
                              <Button onClick={()=>handleDownload(data.missing_api_music_data,'MissingMusicData')}>
                               Download
@@ -135,6 +145,8 @@ function Datadebugger() {
                            <TableCell>{data ? data.existing_season_count:''}</TableCell>
                            <TableCell>{data ? data.api_season_count:''}</TableCell>
                            <TableCell>{data ? data.api_season_count-data.existing_season_count:''}</TableCell>
+                            <TableCell></TableCell>
+                            <TableCell></TableCell>
                            <TableCell>{data  &&  data.api_season_count>data.existing_season_count ?       
                              <Button onClick={()=>handleDownload(data.missing_api_season_data,'MissingSeasonData')}>
                               Download
@@ -148,6 +160,8 @@ function Datadebugger() {
                            <TableCell>{data ? data.existing_episode_count:''}</TableCell>
                            <TableCell>{data ? data.api_episode_count:''}</TableCell>
                            <TableCell>{data ? data.api_episode_count-data.existing_episode_count:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.EpisodeCount:''}</TableCell>
+                            <TableCell>{data ? data.count_from_api.EpisodeCount-data.existing_episode_count:''}</TableCell>
                            <TableCell>{data  &&  data.api_episode_count>data.existing_episode_count ?       
                              <Button onClick={()=>handleDownload(data.missing_api_episode_data,'MissingEpisodeData')}>
                               Download