feat: add onlinefix scraper

package.json

@@ -39,6 +39,7 @@
     "@types/react-dom": "^18.2.22",
     "@types/uuid": "^9.0.8",
     "@types/webtorrent": "^0.109.8",
+    "@types/windows-1251": "^0.1.22",
     "@typescript-eslint/eslint-plugin": "^7.3.1",
     "@typescript-eslint/parser": "^7.3.1",
     "@vanilla-extract/webpack-plugin": "^2.3.7",
@@ -76,15 +77,17 @@
     "classnames": "^2.5.1",
     "color": "^4.2.3",
     "color.js": "^1.2.0",
-    "date-fns": "^3.5.0",
+    "date-fns": "^3.6.0",
     "electron-squirrel-startup": "^1.0.0",
     "flexsearch": "^0.7.43",
+    "got-scraping": "^4.0.5",
     "i18next": "^23.10.1",
     "i18next-browser-languagedetector": "^7.2.0",
     "jsdom": "^24.0.0",
     "lodash": "^4.17.21",
-    "parse-torrent": "9.1.5",
+    "parse-torrent": "^9.1.5",
     "pretty-bytes": "^6.1.1",
+    "qs": "^6.12.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-i18next": "^14.1.0",
@@ -93,11 +96,14 @@
     "react-router-dom": "^6.22.3",
     "sqlite3": "^5.1.7",
     "systeminformation": "^5.22.3",
+    "tough-cookie": "^4.1.3",
     "typeorm": "^0.3.20",
     "update-electron-app": "^3.0.0",
     "uuid": "^9.0.1",
     "vite-plugin-svgr": "^4.2.0",
     "vite-tsconfig-paths": "^4.3.2",
-    "winston": "^3.12.0"
+    "windows-1251": "^3.0.4",
+    "winston": "^3.12.0",
+    "zod": "^3.22.4"
   }
-}
+}

src/main/constants.ts

@@ -16,6 +16,7 @@ export const repackers = [
   "CPG",
   "TinyRepacks",
   "GOG",
+  "onlinefix",
 ] as const;
 
 export const months = [

src/main/events/helpers/search-games.ts

@@ -13,11 +13,15 @@ const repacksIndex = new Index();
 const repacks = stateManager.getValue("repacks");
 
 for (let i = 0; i < repacks.length; i++) {
-  const repack = repacks[i];
+  try {
-  const formatter =
+    const repack = repacks[i];
-    repackerFormatter[repack.repacker as keyof typeof repackerFormatter];
+    const formatter =
+      repackerFormatter[repack.repacker as keyof typeof repackerFormatter];
 
-  repacksIndex.add(i, formatName(formatter(repack.title)));
+    repacksIndex.add(i, formatName(formatter(repack.title)));
+  } catch (err) {
+    console.log(err);
+  }
 }
 
 export const HITS_PER_PAGE = 12;

src/main/helpers/formatters.ts

@@ -47,6 +47,7 @@ export const xatabFormatter = (title: string) =>
     .replace(/(v\.?([0-9]| )+)+([0-9]|\.|-|_|\/|[a-zA-Z]| )+/, "");
 
 export const tinyRepacksFormatter = (title: string) => title;
+export const onlinefixFormatter = (title: string) => title;
 
 export const gogFormatter = (title: string) =>
   title.replace(/(v\.[0-9]+|v[0-9]+\.|v[0-9]{4})+.+/, "");

src/main/helpers/index.ts

@@ -11,6 +11,7 @@ import {
   xatabFormatter,
   tinyRepacksFormatter,
   gogFormatter,
+  onlinefixFormatter,
 } from "./formatters";
 import { months, repackers } from "../constants";
 
@@ -40,6 +41,7 @@ export const repackerFormatter: Record<
   CPG: (title: string) => title,
   TinyRepacks: tinyRepacksFormatter,
   GOG: gogFormatter,
+  onlinefix: onlinefixFormatter,
 };
 
 export const formatUploadDate = (str: string) => {

src/main/index.ts

@@ -19,6 +19,7 @@ import { Repack } from "./entity";
 import { Notification } from "electron";
 import { t } from "i18next";
 import { In } from "typeorm";
+import { getNewRepacksFromOnlineFix } from "./services/repack-tracker/online-fix";
 
 TorrentClient.startTorrentClient(writePipe.socketPath, readPipe.socketPath);
 
@@ -74,6 +75,9 @@ const checkForNewRepacks = async () => {
     getNewRepacksFromCPG(
       existingRepacks.filter((repack) => repack.repacker === "CPG")
     ),
+    getNewRepacksFromOnlineFix(
+      existingRepacks.filter((repack) => repack.repacker === "onlinefix")
+    ),
     track1337xUsers(existingRepacks),
   ]).then(() => {
     repackRepository.count().then((count) => {

src/main/services/repack-tracker/online-fix.ts

@@ -0,0 +1,186 @@
+import { Repack } from "@main/entity";
+import { savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+import { logger } from "../logger";
+import { stringify } from "qs";
+import { z } from "zod";
+import parseTorrent, { toMagnetURI } from "parse-torrent";
+import { JSDOM } from "jsdom";
+import { gotScraping } from "got-scraping";
+import { CookieJar } from "tough-cookie";
+
+import { format, parse, sub } from "date-fns";
+import { ru } from "date-fns/locale";
+import { decode } from "windows-1251";
+
+const preLoginSchema = z.object({
+  field: z.string(),
+  value: z.string(),
+});
+
+export const getNewRepacksFromOnlineFix = async (
+  existingRepacks: Repack[] = []
+): Promise<void> => {
+  const hasCredentials =
+    process.env.ONLINEFIX_USERNAME && process.env.ONLINEFIX_PASSWORD;
+  if (hasCredentials) return;
+
+  const http = gotScraping.extend({
+    headerGeneratorOptions: {
+      browsers: [
+        {
+          name: "chrome",
+          minVersion: 87,
+          maxVersion: 89,
+        },
+      ],
+      devices: ["desktop"],
+      locales: ["en-US"],
+      operatingSystems: ["windows", "linux"],
+    },
+    cookieJar: new CookieJar(),
+  });
+
+  await http.get("https://online-fix.me/");
+  const preLogin = await http
+    .get("https://online-fix.me/engine/ajax/authtoken.php", {
+      headers: {
+        "X-Requested-With": "XMLHttpRequest",
+        Referer: "https://online-fix.me/",
+      },
+    })
+    .json();
+
+  const parsedPreLoginRes = preLoginSchema.parse(preLogin);
+  const tokenField = parsedPreLoginRes.field;
+  const tokenValue = parsedPreLoginRes.value;
+
+  const login = await http
+    .post("https://online-fix.me/", {
+      encoding: "binary",
+      headers: {
+        Referer: "https://online-fix.me",
+        Origin: "https://online-fix.me",
+        "Content-Type": "application/x-www-form-urlencoded",
+      },
+      body: stringify({
+        login_name: process.env.ONLINEFIX_USERNAME,
+        login_password: process.env.ONLINEFIX_PASSWORD,
+        login: "submit",
+        [tokenField]: tokenValue,
+      }),
+    })
+    .text();
+
+  const dom = new JSDOM(login);
+  const document = dom.window.document;
+  const repacks: GameRepackInput[] = [];
+
+  try {
+    const articles = Array.from(document.querySelectorAll(".news"));
+
+    for await (const article of articles) {
+      const gameName = decode(
+        article.querySelector("h2.title")?.textContent?.trim()
+      )
+        .replace("по сети", "")
+        .trim();
+
+      const gameLink = article.querySelector("a")?.getAttribute("href");
+
+      if (!gameLink) continue;
+
+      const gamePage = await http
+        .get(gameLink, {
+          encoding: "binary",
+        })
+        .text();
+
+      const gameDom = new JSDOM(gamePage);
+      const gameDocument = gameDom.window.document;
+
+      const uploadDateText = gameDocument.querySelector(
+        "#dle-content > div > article > div.full-story-header.wide-block.clr > div.full-story-top-panel.clr > div.date.left > time"
+      ).textContent;
+
+      let decodedDateText = decode(uploadDateText);
+
+      // "Вчера" significa ontem.
+      if (decodedDateText.includes("Вчера")) {
+        const yesterday = sub(new Date(), { days: 1 });
+        const formattedYesterday = format(yesterday, "d LLLL yyyy", {
+          locale: ru,
+        });
+        decodedDateText = decodedDateText.replace("Вчера", formattedYesterday);
+      }
+
+      const uploadDate = parse(
+        decodedDateText,
+        "d LLLL yyyy, HH:mm",
+        new Date(),
+        {
+          locale: ru,
+        }
+      );
+
+      const torrentButtons = Array.from(
+        gameDocument.querySelectorAll("a")
+      ).filter((a) => a.textContent?.includes("Torrent"));
+
+      const torrentPrePage = torrentButtons[0]?.getAttribute("href");
+      if (!torrentPrePage) continue;
+
+      const torrentPage = await http
+        .get(torrentPrePage, {
+          encoding: "binary",
+          headers: {
+            Referer: gameLink,
+          },
+        })
+        .text();
+
+      const torrentDom = new JSDOM(torrentPage);
+      const torrentDocument = torrentDom.window.document;
+
+      const torrentLink = torrentDocument
+        .querySelector("a:nth-child(2)")
+        ?.getAttribute("href");
+
+      const torrentFile = Buffer.from(
+        await http
+          .get(`${torrentPrePage}/${torrentLink}`, {
+            responseType: "buffer",
+          })
+          .buffer()
+      );
+
+      const torrent = parseTorrent(torrentFile);
+      const magnetLink = toMagnetURI({
+        infoHash: torrent.infoHash,
+      });
+
+      repacks.push({
+        fileSize: "NA",
+        magnet: magnetLink,
+        page: 1,
+        repacker: "onlinefix",
+        title: gameName,
+        uploadDate: uploadDate,
+      });
+    }
+  } catch (err) {
+    logger.error(err.message, { method: "getNewRepacksFromOnlineFix" });
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+};