fix: change onlinefixScraper to fetch

src/main/main.ts

@@ -5,7 +5,7 @@ import {
   getNewRepacksFromCPG,
   getNewRepacksFromUser,
   getNewRepacksFromXatab,
-  // getNewRepacksFromOnlineFix,
+  getNewRepacksFromOnlineFix,
   readPipe,
   startProcessWatcher,
   writePipe,
@@ -79,9 +79,9 @@ const checkForNewRepacks = async () => {
     getNewRepacksFromCPG(
       existingRepacks.filter((repack) => repack.repacker === "CPG")
     ),
-    // getNewRepacksFromOnlineFix(
-    //   existingRepacks.filter((repack) => repack.repacker === "onlinefix")
-    // ),
+    getNewRepacksFromOnlineFix(
+      existingRepacks.filter((repack) => repack.repacker === "onlinefix")
+    ),
     track1337xUsers(existingRepacks),
   ]).then(() => {
     repackRepository.count().then((count) => {

src/main/services/repack-tracker/helpers.ts

@@ -20,3 +20,19 @@ export const requestWebPage = async (url: string) => {
     },
   }).then((response) => response.text());
 };
+
+export const decodeNonUtf8Response = async (res: Response) => {
+  const contentType = res.headers.get("content-type");
+  if (!contentType) return res.text();
+
+  const charset = contentType.substring(contentType.indexOf("charset=") + 8);
+
+  const text = await res.arrayBuffer().then((ab) => {
+    const dataView = new DataView(ab);
+    const decoder = new TextDecoder(charset);
+
+    return decoder.decode(dataView);
+  });
+
+  return text;
+};

src/main/services/repack-tracker/index.ts

@@ -2,4 +2,4 @@ export * from "./1337x";
 export * from "./xatab";
 export * from "./cpg-repacks";
 export * from "./gog";
-// export * from "./online-fix";
+export * from "./online-fix";

src/main/services/repack-tracker/online-fix.ts

@@ -1,56 +1,42 @@
 import { Repack } from "@main/entity";
-import { savePage } from "./helpers";
+import { decodeNonUtf8Response, savePage } from "./helpers";
 import { logger } from "../logger";
 import parseTorrent, {
   toMagnetURI,
   Instance as TorrentInstance,
 } from "parse-torrent";
 import { JSDOM } from "jsdom";
-import { gotScraping } from "got-scraping";
-import { CookieJar } from "tough-cookie";
 
 import { format, parse, sub } from "date-fns";
 import { ru } from "date-fns/locale";
 import { decode } from "windows-1251";
 import { onlinefixFormatter } from "@main/helpers";
+import makeFetchCookie from "fetch-cookie";
+import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
 
 export const getNewRepacksFromOnlineFix = async (
   existingRepacks: Repack[] = [],
   page = 1,
-  cookieJar = new CookieJar()
+  cookieJar = new makeFetchCookie.toughCookie.CookieJar()
 ): Promise<void> => {
   const hasCredentials =
     import.meta.env.MAIN_VITE_ONLINEFIX_USERNAME &&
     import.meta.env.MAIN_VITE_ONLINEFIX_PASSWORD;
   if (!hasCredentials) return;
 
-  const http = gotScraping.extend({
-    headerGeneratorOptions: {
-      browsers: [
-        {
-          name: "chrome",
-          minVersion: 87,
-          maxVersion: 89,
-        },
-      ],
-      devices: ["desktop"],
-      locales: ["en-US"],
-      operatingSystems: ["windows", "linux"],
-    },
-    cookieJar: cookieJar,
-  });
+  const http = makeFetchCookie(fetch, cookieJar);
 
   if (page === 1) {
-    await http.get("https://online-fix.me/");
+    await http("https://online-fix.me/");
+
     const preLogin =
-      ((await http
-        .get("https://online-fix.me/engine/ajax/authtoken.php", {
-          headers: {
-            "X-Requested-With": "XMLHttpRequest",
-            Referer: "https://online-fix.me/",
-          },
-        })
-        .json()) as {
+      ((await http("https://online-fix.me/engine/ajax/authtoken.php", {
+        method: "GET",
+        headers: {
+          "X-Requested-With": "XMLHttpRequest",
+          Referer: "https://online-fix.me/",
+        },
+      }).then((res) => res.json())) as {
         field: string;
         value: string;
       }) || undefined;
@@ -64,27 +50,25 @@ export const getNewRepacksFromOnlineFix = async (
       [preLogin.field]: preLogin.value,
     });
 
-    await http
-      .post("https://online-fix.me/", {
-        encoding: "binary",
-        headers: {
-          Referer: "https://online-fix.me",
-          Origin: "https://online-fix.me",
-          "Content-Type": "application/x-www-form-urlencoded",
-        },
-        body: params.toString(),
-      })
-      .text();
+    await http("https://online-fix.me/", {
+      method: "POST",
+      headers: {
+        Referer: "https://online-fix.me",
+        Origin: "https://online-fix.me",
+        "Content-Type": "application/x-www-form-urlencoded",
+      },
+      body: params.toString(),
+    });
   }
 
   const pageParams = page > 1 ? `${`/page/${page}`}` : "";
 
-  const home = await http.get(`https://online-fix.me${pageParams}`, {
-    encoding: "binary",
-  });
-  const document = new JSDOM(home.body).window.document;
+  const home = await http(`https://online-fix.me${pageParams}`).then((res) =>
+    decodeNonUtf8Response(res)
+  );
+  const document = new JSDOM(home).window.document;
 
-  const repacks = [];
+  const repacks: QueryDeepPartialEntity<Repack>[] = [];
   const articles = Array.from(document.querySelectorAll(".news"));
   const totalPages = Number(
     document.querySelector("nav > a:nth-child(13)")?.textContent
@@ -93,23 +77,21 @@ export const getNewRepacksFromOnlineFix = async (
   try {
     await Promise.all(
       articles.map(async (article) => {
-        const gameName = onlinefixFormatter(
-          decode(article.querySelector("h2.title")?.textContent?.trim())
-        );
+        const gameText = article.querySelector("h2.title")?.textContent?.trim();
+        if (!gameText) return;
+
+        const gameName = onlinefixFormatter(gameText);
 
         const gameLink = article.querySelector("a")?.getAttribute("href");
-
         if (!gameLink) return;
 
-        const gamePage = await http
-          .get(gameLink, {
-            encoding: "binary",
-          })
-          .text();
-
+        const gamePage = await http(gameLink).then((res) =>
+          decodeNonUtf8Response(res)
+        );
         const gameDocument = new JSDOM(gamePage).window.document;
 
-        const uploadDateText = gameDocument.querySelector("time").textContent;
+        const uploadDateText = gameDocument.querySelector("time")?.textContent;
+        if (!uploadDateText) return;
 
         let decodedDateText = decode(uploadDateText);
 
@@ -141,14 +123,11 @@ export const getNewRepacksFromOnlineFix = async (
         const torrentPrePage = torrentButtons[0]?.getAttribute("href");
         if (!torrentPrePage) return;
 
-        const torrentPage = await http
-          .get(torrentPrePage, {
-            encoding: "binary",
-            headers: {
-              Referer: gameLink,
-            },
-          })
-          .text();
+        const torrentPage = await http(torrentPrePage, {
+          headers: {
+            Referer: gameLink,
+          },
+        }).then((res) => res.text());
 
         const torrentDocument = new JSDOM(torrentPage).window.document;
 
@@ -157,11 +136,9 @@ export const getNewRepacksFromOnlineFix = async (
           ?.getAttribute("href");
 
         const torrentFile = Buffer.from(
-          await http
-            .get(`${torrentPrePage}/${torrentLink}`, {
-              responseType: "buffer",
-            })
-            .buffer()
+          await http(`${torrentPrePage}/${torrentLink}`).then((res) =>
+            res.arrayBuffer()
+          )
         );
 
         const torrent = parseTorrent(torrentFile) as TorrentInstance;
@@ -170,6 +147,8 @@ export const getNewRepacksFromOnlineFix = async (
         });
 
         const torrentSizeInBytes = torrent.length;
+        if (!torrentSizeInBytes) return;
+
         const fileSizeFormatted =
           torrentSizeInBytes >= 1024 ** 3
             ? `${(torrentSizeInBytes / 1024 ** 3).toFixed(1)}GBs`