feat: adding auto refresh of download sources

src/main/entity/download-source.entity.ts

@@ -7,6 +7,7 @@ import {
   OneToMany,
 } from "typeorm";
 import { Repack } from "./repack.entity";
+import { DownloadSourceStatus } from "@shared";
 
 @Entity("download_source")
 export class DownloadSource {
@@ -19,6 +20,12 @@ export class DownloadSource {
   @Column("text")
   name: string;
 
+  @Column("text", { nullable: true })
+  etag: string | null;
+
+  @Column("text", { default: "online" })
+  status: DownloadSourceStatus;
+
   @OneToMany(() => Repack, (repack) => repack.downloadSource, { cascade: true })
   repacks: Repack[];
 

src/main/events/download-sources/add-download-source.ts

@@ -1,12 +1,11 @@
 import { registerEvent } from "../register-event";
-import { chunk } from "lodash-es";
 import { dataSource } from "@main/data-source";
-import { DownloadSource, Repack } from "@main/entity";
+import { DownloadSource } from "@main/entity";
 import axios from "axios";
-import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
 import { downloadSourceSchema } from "../helpers/validators";
 import { repackRepository } from "@main/repository";
 import { repacksWorker } from "@main/workers";
+import { insertDownloadsFromSource } from "@main/helpers";
 
 const addDownloadSource = async (
   _event: Electron.IpcMainInvokeEvent,
@@ -22,30 +21,12 @@ const addDownloadSource = async (
         .getRepository(DownloadSource)
         .save({ url, name: source.name });
 
-      const repacks: QueryDeepPartialEntity<Repack>[] = source.downloads.map(
-        (download) => ({
-          title: download.title,
-          magnet: download.uris[0],
-          fileSize: download.fileSize,
-          repacker: source.name,
-          uploadDate: download.uploadDate,
-          downloadSource: { id: downloadSource.id },
-        })
+      await insertDownloadsFromSource(
+        transactionalEntityManager,
+        downloadSource,
+        source.downloads
       );
 
-      const downloadsChunks = chunk(repacks, 800);
-
-      for (const chunk of downloadsChunks) {
-        await transactionalEntityManager
-          .getRepository(Repack)
-          .createQueryBuilder()
-          .insert()
-          .values(chunk)
-          .updateEntity(false)
-          .orIgnore()
-          .execute();
-      }
-
       return downloadSource;
     }
   );

src/main/helpers/download-source.ts

@@ -0,0 +1,69 @@
+import { dataSource } from "@main/data-source";
+import { DownloadSource, Repack } from "@main/entity";
+import { downloadSourceSchema } from "@main/events/helpers/validators";
+import { downloadSourceRepository } from "@main/repository";
+import { downloadSourceWorker } from "@main/workers";
+import { chunk } from "lodash-es";
+import type { EntityManager } from "typeorm";
+import type { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
+import { z } from "zod";
+
+export const insertDownloadsFromSource = async (
+  trx: EntityManager,
+  downloadSource: DownloadSource,
+  downloads: z.infer<typeof downloadSourceSchema>["downloads"]
+) => {
+  const repacks: QueryDeepPartialEntity<Repack>[] = downloads.map(
+    (download) => ({
+      title: download.title,
+      magnet: download.uris[0],
+      fileSize: download.fileSize,
+      repacker: downloadSource.name,
+      uploadDate: download.uploadDate,
+      downloadSource: { id: downloadSource.id },
+    })
+  );
+
+  const downloadsChunks = chunk(repacks, 800);
+
+  for (const chunk of downloadsChunks) {
+    await trx
+      .getRepository(Repack)
+      .createQueryBuilder()
+      .insert()
+      .values(chunk)
+      .updateEntity(false)
+      .orIgnore()
+      .execute();
+  }
+};
+
+export const fetchDownloadSourcesAndUpdate = async () => {
+  const downloadSources = await downloadSourceRepository.find({
+    order: {
+      id: "desc",
+    },
+  });
+
+  const results = await downloadSourceWorker.run(downloadSources, {
+    name: "getUpdatedRepacks",
+  });
+
+  await dataSource.transaction(async (transactionalEntityManager) => {
+    for (const result of results) {
+      await transactionalEntityManager.getRepository(DownloadSource).update(
+        { id: result.id },
+        {
+          etag: result.etag,
+          status: result.status,
+        }
+      );
+
+      await insertDownloadsFromSource(
+        transactionalEntityManager,
+        result,
+        result.downloads
+      );
+    }
+  });
+};

src/main/helpers/index.ts

@@ -58,3 +58,4 @@ export const requestWebPage = async (url: string) => {
 };
 
 export * from "./ps";
+export * from "./download-source";

src/main/main.ts

@@ -8,6 +8,7 @@ import { UserPreferences } from "./entity";
 import { RealDebridClient } from "./services/real-debrid";
 import { Not } from "typeorm";
 import { repacksWorker } from "./workers";
+import { fetchDownloadSourcesAndUpdate } from "./helpers";
 
 startMainLoop();
 
@@ -27,15 +28,14 @@ const loadState = async (userPreferences: UserPreferences | null) => {
 
   if (game) DownloadManager.startDownload(game);
 
-  repackRepository
-    .find({
-      order: {
-        createdAt: "DESC",
-      },
-    })
-    .then((repacks) => {
-      repacksWorker.run(repacks, { name: "setRepacks" });
-    });
+  const repacks = await repackRepository.find({
+    order: {
+      createdAt: "DESC",
+    },
+  });
+
+  repacksWorker.run(repacks, { name: "setRepacks" });
+  fetchDownloadSourcesAndUpdate();
 };
 
 userPreferencesRepository

src/main/workers/download-source.worker.ts

@@ -0,0 +1,49 @@
+import { downloadSourceSchema } from "@main/events/helpers/validators";
+import { DownloadSourceStatus } from "@shared";
+import type { DownloadSource } from "@types";
+import axios, { AxiosError, AxiosHeaders } from "axios";
+import { z } from "zod";
+
+export type DownloadSourceResponse = z.infer<typeof downloadSourceSchema> & {
+  etag: string | null;
+  status: DownloadSourceStatus;
+};
+
+export const getUpdatedRepacks = async (downloadSources: DownloadSource[]) => {
+  const results: DownloadSourceResponse[] = [];
+
+  for (const downloadSource of downloadSources) {
+    const headers = new AxiosHeaders();
+
+    if (downloadSource.etag) {
+      headers.set("If-None-Match", downloadSource.etag);
+    }
+
+    try {
+      const response = await axios.get(downloadSource.url, {
+        headers,
+      });
+
+      const source = downloadSourceSchema.parse(response.data);
+
+      results.push({
+        ...downloadSource,
+        downloads: source.downloads,
+        etag: response.headers["etag"],
+        status: DownloadSourceStatus.UpToDate,
+      });
+    } catch (err: unknown) {
+      const isNotModified = (err as AxiosError).response?.status === 304;
+
+      results.push({
+        ...downloadSource,
+        downloads: [],
+        status: isNotModified
+          ? DownloadSourceStatus.UpToDate
+          : DownloadSourceStatus.Errored,
+      });
+    }
+  }
+
+  return results;
+};

src/main/workers/index.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import steamGamesWorkerPath from "./steam-games.worker?modulePath";
 import repacksWorkerPath from "./repacks.worker?modulePath";
+import downloadSourceWorkerPath from "./download-source.worker?modulePath";
 
 import Piscina from "piscina";
 
@@ -16,3 +17,7 @@ export const steamGamesWorker = new Piscina({
 export const repacksWorker = new Piscina({
   filename: repacksWorkerPath,
 });
+
+export const downloadSourceWorker = new Piscina({
+  filename: downloadSourceWorkerPath,
+});