Merge remote-tracking branch 'upstream/main' into feature/better-repack-modal

src/main/services/downloaders/torrent.downloader.ts

@@ -1,7 +1,6 @@
 import path from "node:path";
 import cp from "node:child_process";
 import fs from "node:fs";
-import * as Sentry from "@sentry/electron/main";
 import { app, dialog } from "electron";
 import type { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
 
@@ -87,8 +86,6 @@ export class TorrentDownloader extends Downloader {
           downloadSpeed: payload.downloadSpeed,
           timeRemaining: payload.timeRemaining,
         });
-      } catch (err) {
-        Sentry.captureException(err);
       } finally {
         await new Promise((resolve) => setTimeout(resolve, 100));
       }

src/main/services/repack-tracker/cpg-repacks.ts

@@ -1,64 +0,0 @@
-import { JSDOM } from "jsdom";
-
-import { Repack } from "@main/entity";
-
-import { requestWebPage, savePage } from "./helpers";
-import { logger } from "../logger";
-import type { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
-
-export const getNewRepacksFromCPG = async (
-  existingRepacks: Repack[] = [],
-  page = 1
-): Promise<void> => {
-  const data = await requestWebPage(`https://cpgrepacks.site/page/${page}`);
-
-  const { window } = new JSDOM(data);
-
-  const repacks: QueryDeepPartialEntity<Repack>[] = [];
-
-  try {
-    Array.from(window.document.querySelectorAll(".post")).forEach(($post) => {
-      const $title = $post.querySelector(".entry-title")!;
-      const uploadDate = $post.querySelector("time")?.getAttribute("datetime");
-
-      const $downloadInfo = Array.from(
-        $post.querySelectorAll(".wp-block-heading")
-      ).find(($heading) => $heading.textContent?.startsWith("Download"));
-
-      /* Side note: CPG often misspells "Magnet" as "Magent" */
-      const $magnet = Array.from($post.querySelectorAll("a")).find(
-        ($a) =>
-          $a.textContent?.startsWith("Magnet") ||
-          $a.textContent?.startsWith("Magent")
-      );
-
-      const fileSize = ($downloadInfo?.textContent ?? "")
-        .split("Download link => ")
-        .at(1);
-
-      repacks.push({
-        title: $title.textContent!,
-        fileSize: fileSize ?? "N/A",
-        magnet: $magnet!.href,
-        repacker: "CPG",
-        page,
-        uploadDate: uploadDate ? new Date(uploadDate) : new Date(),
-      });
-    });
-  } catch (err: unknown) {
-    logger.error((err as Error).message, { method: "getNewRepacksFromCPG" });
-  }
-
-  const newRepacks = repacks.filter(
-    (repack) =>
-      !existingRepacks.some(
-        (existingRepack) => existingRepack.title === repack.title
-      )
-  );
-
-  if (!newRepacks.length) return;
-
-  await savePage(newRepacks);
-
-  return getNewRepacksFromCPG(existingRepacks, page + 1);
-};

src/main/services/repack-tracker/gog.ts

@@ -6,32 +6,57 @@ import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity
 
 const virtualConsole = new VirtualConsole();
 
+const getUploadDate = (document: Document) => {
+  const $modifiedTime = document.querySelector(
+    '[property="article:modified_time"]'
+  ) as HTMLMetaElement;
+  if ($modifiedTime) return $modifiedTime.content;
+
+  const $publishedTime = document.querySelector(
+    '[property="article:published_time"]'
+  ) as HTMLMetaElement;
+  return $publishedTime.content;
+};
+
+const getDownloadLink = (document: Document) => {
+  const $latestDownloadButton = document.querySelector(
+    ".download-btn:not(.lightweight-accordion *)"
+  ) as HTMLAnchorElement;
+  if ($latestDownloadButton) return $latestDownloadButton.href;
+
+  const $downloadButton = document.querySelector(
+    ".download-btn"
+  ) as HTMLAnchorElement;
+  if (!$downloadButton) return null;
+
+  return $downloadButton.href;
+};
+
+const getMagnet = (downloadLink: string) => {
+  if (downloadLink.startsWith("http")) {
+    const { searchParams } = new URL(downloadLink);
+    return Buffer.from(searchParams.get("url")!, "base64").toString("utf-8");
+  }
+
+  return downloadLink;
+};
+
 const getGOGGame = async (url: string) => {
   const data = await requestWebPage(url);
   const { window } = new JSDOM(data, { virtualConsole });
 
-  const $modifiedTime = window.document.querySelector(
-    '[property="article:modified_time"]'
-  ) as HTMLMetaElement;
+  const downloadLink = getDownloadLink(window.document);
+  if (!downloadLink) return null;
 
-  const $em = window.document.querySelector(
-    "p:not(.lightweight-accordion *) em"
-  )!;
+  const $em = window.document.querySelector("p em");
+  if (!$em) return null;
   const fileSize = $em.textContent!.split("Size: ").at(1);
-  const $downloadButton = window.document.querySelector(
-    ".download-btn:not(.lightweight-accordion *)"
-  ) as HTMLAnchorElement;
-
-  const { searchParams } = new URL($downloadButton.href);
-  const magnet = Buffer.from(searchParams.get("url")!, "base64").toString(
-    "utf-8"
-  );
 
   return {
     fileSize: fileSize ?? "N/A",
-    uploadDate: new Date($modifiedTime.content),
+    uploadDate: new Date(getUploadDate(window.document)),
     repacker: "GOG",
-    magnet,
+    magnet: getMagnet(downloadLink),
     page: 1,
   };
 };
@@ -62,7 +87,7 @@ export const getNewGOGGames = async (existingRepacks: Repack[] = []) => {
       if (!gameExists) {
         const game = await getGOGGame(href);
 
-        repacks.push({ ...game, title });
+        if (game) repacks.push({ ...game, title });
       }
     }
 

src/main/services/repack-tracker/helpers.ts

@@ -1,3 +1,4 @@
+import axios from "axios";
 import UserAgent from "user-agents";
 
 import type { Repack } from "@main/entity";
@@ -13,12 +14,13 @@ export const savePage = async (repacks: QueryDeepPartialEntity<Repack>[]) =>
 export const requestWebPage = async (url: string) => {
   const userAgent = new UserAgent();
 
-  return fetch(url, {
-    method: "GET",
-    headers: {
-      "User-Agent": userAgent.toString(),
-    },
-  }).then((response) => response.text());
+  return axios
+    .get(url, {
+      headers: {
+        "User-Agent": userAgent.toString(),
+      },
+    })
+    .then((response) => response.data);
 };
 
 export const decodeNonUtf8Response = async (res: Response) => {

src/main/services/repack-tracker/index.ts

@@ -1,5 +1,4 @@
 export * from "./1337x";
 export * from "./xatab";
-export * from "./cpg-repacks";
 export * from "./gog";
 export * from "./online-fix";

src/main/services/repack-tracker/online-fix.ts

@@ -1,15 +1,16 @@
 import { Repack } from "@main/entity";
 import { decodeNonUtf8Response, savePage } from "./helpers";
 import { logger } from "../logger";
-import parseTorrent, {
-  toMagnetURI,
-  Instance as TorrentInstance,
-} from "parse-torrent";
 import { JSDOM } from "jsdom";
 
 import { format, parse, sub } from "date-fns";
 import { ru } from "date-fns/locale";
 
+import createWorker from "@main/workers/torrent-parser.worker?nodeWorker";
+import { toMagnetURI } from "parse-torrent";
+
+const worker = createWorker({});
+
 import { onlinefixFormatter } from "@main/helpers";
 import makeFetchCookie from "fetch-cookie";
 import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
@@ -139,27 +140,23 @@ export const getNewRepacksFromOnlineFix = async (
           ?.getAttribute("href");
 
         const torrentFile = Buffer.from(
-          await http(`${torrentPrePage}/${torrentLink}`).then((res) =>
+          await http(`${torrentPrePage}${torrentLink}`).then((res) =>
             res.arrayBuffer()
           )
         );
 
-        const torrent = parseTorrent(torrentFile) as TorrentInstance;
-        const magnetLink = toMagnetURI({
-          infoHash: torrent.infoHash,
+        worker.once("message", (torrent) => {
+          repacks.push({
+            fileSize: formatBytes(torrent.length ?? 0),
+            magnet: toMagnetURI(torrent),
+            page: 1,
+            repacker: "onlinefix",
+            title: gameName,
+            uploadDate: uploadDate,
+          });
         });
 
-        const torrentSizeInBytes = torrent.length;
-        if (!torrentSizeInBytes) return;
-
-        repacks.push({
-          fileSize: formatBytes(torrentSizeInBytes),
-          magnet: magnetLink,
-          page: 1,
-          repacker: "onlinefix",
-          title: gameName,
-          uploadDate: uploadDate,
-        });
+        worker.postMessage(torrentFile);
       })
     );
   } catch (err: unknown) {

src/main/services/repack-tracker/xatab.ts

@@ -9,6 +9,7 @@ import { toMagnetURI } from "parse-torrent";
 import type { Instance } from "parse-torrent";
 import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
 import { formatBytes } from "@shared";
+import { getFileBuffer } from "@main/helpers";
 
 const worker = createWorker({});
 
@@ -27,7 +28,7 @@ const formatXatabDate = (str: string) => {
 
 const getXatabRepack = (
   url: string
-): Promise<{ fileSize: string; magnet: string; uploadDate: Date }> => {
+): Promise<{ fileSize: string; magnet: string; uploadDate: Date } | null> => {
   return new Promise((resolve) => {
     (async () => {
       const data = await requestWebPage(url);
@@ -40,15 +41,20 @@ const getXatabRepack = (
         ".download-torrent"
       ) as HTMLAnchorElement;
 
-      if (!$downloadButton) throw new Error("Download button not found");
+      if (!$downloadButton) return resolve(null);
+
+      worker.once("message", (torrent: Instance | null) => {
+        if (!torrent) return resolve(null);
 
-      worker.once("message", (torrent: Instance) => {
         resolve({
           fileSize: formatBytes(torrent.length ?? 0),
           magnet: toMagnetURI(torrent),
           uploadDate: formatXatabDate($uploadDate!.textContent!),
         });
       });
+
+      const buffer = await getFileBuffer($downloadButton.href);
+      worker.postMessage(buffer);
     })();
   });
 };
@@ -69,12 +75,14 @@ export const getNewRepacksFromXatab = async (
     try {
       const repack = await getXatabRepack(($a as HTMLAnchorElement).href);
 
-      repacks.push({
-        title: $a.textContent!,
-        repacker: "Xatab",
-        ...repack,
-        page,
-      });
+      if (repack) {
+        repacks.push({
+          title: $a.textContent!,
+          repacker: "Xatab",
+          ...repack,
+          page,
+        });
+      }
     } catch (err: unknown) {
       logger.error((err as Error).message, {
         method: "getNewRepacksFromXatab",

src/main/services/update-resolver.ts

@@ -3,107 +3,9 @@ import { app } from "electron";
 
 import { chunk } from "lodash-es";
 
-import { createDataSource, dataSource } from "@main/data-source";
-import { Repack, RepackerFriendlyName, SteamGame } from "@main/entity";
-import {
-  migrationScriptRepository,
-  repackRepository,
-  repackerFriendlyNameRepository,
-  steamGameRepository,
-} from "@main/repository";
-import { MigrationScript } from "@main/entity/migration-script.entity";
-import { Like } from "typeorm";
-
-const migrationScripts = {
-  /*
-    0.0.6 -> 0.0.7
-    Xatab repacks were previously created with an incorrect upload date.
-    This migration script will update the upload date of all Xatab repacks.
-  */
-  "0.0.7": async (updateRepacks: Repack[]) => {
-    const VERSION = "0.0.7";
-
-    const migrationScript = await migrationScriptRepository.findOne({
-      where: {
-        version: VERSION,
-      },
-    });
-
-    if (!migrationScript) {
-      const xatabRepacks = updateRepacks.filter(
-        (repack) => repack.repacker === "Xatab"
-      );
-
-      await dataSource.transaction(async (transactionalEntityManager) => {
-        await Promise.all(
-          xatabRepacks.map((repack) =>
-            transactionalEntityManager.getRepository(Repack).update(
-              {
-                title: repack.title,
-                repacker: repack.repacker,
-              },
-              {
-                uploadDate: repack.uploadDate,
-              }
-            )
-          )
-        );
-
-        await transactionalEntityManager.getRepository(MigrationScript).insert({
-          version: VERSION,
-        });
-      });
-    }
-  },
-  /*
-    1.0.1 -> 1.1.0
-    A few torrents scraped from 1337x were previously created with an incorrect upload date.
-  */
-  "1.1.0": async () => {
-    const VERSION = "1.1.0";
-
-    const migrationScript = await migrationScriptRepository.findOne({
-      where: {
-        version: VERSION,
-      },
-    });
-
-    if (!migrationScript) {
-      await dataSource.transaction(async (transactionalEntityManager) => {
-        const repacks = await transactionalEntityManager
-          .getRepository(Repack)
-          .find({
-            where: {
-              uploadDate: Like("1%"),
-            },
-          });
-
-        await Promise.all(
-          repacks.map(async (repack) => {
-            return transactionalEntityManager
-              .getRepository(Repack)
-              .update(
-                { id: repack.id },
-                { uploadDate: new Date(repack.uploadDate) }
-              );
-          })
-        );
-
-        await transactionalEntityManager.getRepository(MigrationScript).insert({
-          version: VERSION,
-        });
-      });
-    }
-  },
-};
-
-export const runMigrationScripts = async (updateRepacks: Repack[]) => {
-  return Promise.all(
-    Object.values(migrationScripts).map((migrationScript) => {
-      return migrationScript(updateRepacks);
-    })
-  );
-};
+import { createDataSource } from "@main/data-source";
+import { Repack, SteamGame } from "@main/entity";
+import { repackRepository, steamGameRepository } from "@main/repository";
 
 export const resolveDatabaseUpdates = async () => {
   const updateDataSource = createDataSource({
@@ -114,25 +16,12 @@ export const resolveDatabaseUpdates = async () => {
 
   return updateDataSource.initialize().then(async () => {
     const updateRepackRepository = updateDataSource.getRepository(Repack);
-    const updateRepackerFriendlyNameRepository =
-      updateDataSource.getRepository(RepackerFriendlyName);
     const updateSteamGameRepository = updateDataSource.getRepository(SteamGame);
 
-    const [updateRepacks, updateSteamGames, updateRepackerFriendlyNames] =
-      await Promise.all([
-        updateRepackRepository.find(),
-        updateSteamGameRepository.find(),
-        updateRepackerFriendlyNameRepository.find(),
-      ]);
-
-    await runMigrationScripts(updateRepacks);
-
-    await repackerFriendlyNameRepository
-      .createQueryBuilder()
-      .insert()
-      .values(updateRepackerFriendlyNames)
-      .orIgnore()
-      .execute();
+    const [updateRepacks, updateSteamGames] = await Promise.all([
+      updateRepackRepository.find(),
+      updateSteamGameRepository.find(),
+    ]);
 
     const updateRepacksChunks = chunk(updateRepacks, 800);