Merge pull request #47 from pmenta/feat/onlinefix

Feat/onlinefix

src/main/constants.ts

@@ -16,6 +16,7 @@ export const repackers = [
   "CPG",
   "TinyRepacks",
   "GOG",
+  "onlinefix",
 ] as const;
 
 export const months = [

src/main/helpers/formatters.ts

@@ -49,6 +49,8 @@ export const xatabFormatter = (title: string) =>
     .replace(/(v\.?([0-9]| )+)+([0-9]|\.|-|_|\/|[a-zA-Z]| )+/, "");
 
 export const tinyRepacksFormatter = (title: string) => title;
+export const onlinefixFormatter = (title: string) =>
+  title.replace("по сети", "").trim();
 
 export const gogFormatter = (title: string) =>
   title.replace(/(v\.[0-9]+|v[0-9]+\.|v[0-9]{4})+.+/, "");

src/main/helpers/index.ts

@@ -11,6 +11,7 @@ import {
   xatabFormatter,
   tinyRepacksFormatter,
   gogFormatter,
+  onlinefixFormatter,
 } from "./formatters";
 import { months, repackers } from "../constants";
 
@@ -40,6 +41,7 @@ export const repackerFormatter: Record<
   CPG: (title: string) => title,
   TinyRepacks: tinyRepacksFormatter,
   GOG: gogFormatter,
+  onlinefix: onlinefixFormatter,
 };
 
 export const formatUploadDate = (str: string) => {

src/main/index.ts

@@ -5,6 +5,7 @@ import {
   getNewRepacksFromCPG,
   getNewRepacksFromUser,
   getNewRepacksFromXatab,
+  getNewRepacksFromOnlineFix,
   readPipe,
   startProcessWatcher,
   writePipe,
@@ -77,6 +78,9 @@ const checkForNewRepacks = async () => {
     getNewRepacksFromCPG(
       existingRepacks.filter((repack) => repack.repacker === "CPG")
     ),
+    getNewRepacksFromOnlineFix(
+      existingRepacks.filter((repack) => repack.repacker === "onlinefix")
+    ),
     track1337xUsers(existingRepacks),
   ]).then(() => {
     repackRepository.count().then((count) => {

src/main/services/repack-tracker/index.ts

@@ -2,3 +2,4 @@ export * from "./1337x";
 export * from "./xatab";
 export * from "./cpg-repacks";
 export * from "./gog";
+export * from "./online-fix";

src/main/services/repack-tracker/online-fix.ts

@@ -0,0 +1,208 @@
+import { Repack } from "@main/entity";
+import { savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+import { logger } from "../logger";
+import { stringify } from "qs";
+import parseTorrent, {
+  toMagnetURI,
+  Instance as TorrentInstance,
+} from "parse-torrent";
+import { JSDOM } from "jsdom";
+import { gotScraping } from "got-scraping";
+import { CookieJar } from "tough-cookie";
+
+import { format, parse, sub } from "date-fns";
+import { ru } from "date-fns/locale";
+import { decode } from "windows-1251";
+import { onlinefixFormatter } from "@main/helpers";
+
+export const getNewRepacksFromOnlineFix = async (
+  existingRepacks: Repack[] = [],
+  page = 1,
+  cookieJar = new CookieJar()
+): Promise<void> => {
+  const hasCredentials =
+    process.env.ONLINEFIX_USERNAME && process.env.ONLINEFIX_PASSWORD;
+  if (!hasCredentials) return;
+
+  const http = gotScraping.extend({
+    headerGeneratorOptions: {
+      browsers: [
+        {
+          name: "chrome",
+          minVersion: 87,
+          maxVersion: 89,
+        },
+      ],
+      devices: ["desktop"],
+      locales: ["en-US"],
+      operatingSystems: ["windows", "linux"],
+    },
+    cookieJar: cookieJar,
+  });
+
+  if (page === 1) {
+    await http.get("https://online-fix.me/");
+    const preLogin =
+      ((await http
+        .get("https://online-fix.me/engine/ajax/authtoken.php", {
+          headers: {
+            "X-Requested-With": "XMLHttpRequest",
+            Referer: "https://online-fix.me/",
+          },
+        })
+        .json()) as {
+        field: string;
+        value: string;
+      }) || undefined;
+
+    if (!preLogin.field || !preLogin.value) return;
+
+    const tokenField = preLogin.field;
+    const tokenValue = preLogin.value;
+
+    await http
+      .post("https://online-fix.me/", {
+        encoding: "binary",
+        headers: {
+          Referer: "https://online-fix.me",
+          Origin: "https://online-fix.me",
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: stringify({
+          login_name: process.env.ONLINEFIX_USERNAME,
+          login_password: process.env.ONLINEFIX_PASSWORD,
+          login: "submit",
+          [tokenField]: tokenValue,
+        }),
+      })
+      .text();
+  }
+
+  const pageParams = page > 1 ? `${`/page/${page}`}` : "";
+
+  const home = await http.get(`https://online-fix.me${pageParams}`, {
+    encoding: "binary",
+  });
+  const document = new JSDOM(home.body).window.document;
+
+  const repacks: GameRepackInput[] = [];
+  const articles = Array.from(document.querySelectorAll(".news"));
+  const totalPages = Number(
+    document.querySelector("nav > a:nth-child(13)").textContent
+  );
+
+  try {
+    await Promise.all(
+      articles.map(async (article) => {
+        const gameName = onlinefixFormatter(
+          decode(article.querySelector("h2.title")?.textContent?.trim())
+        );
+
+        const gameLink = article.querySelector("a")?.getAttribute("href");
+
+        if (!gameLink) return;
+
+        const gamePage = await http
+          .get(gameLink, {
+            encoding: "binary",
+          })
+          .text();
+
+        const gameDocument = new JSDOM(gamePage).window.document;
+
+        const uploadDateText = gameDocument.querySelector("time").textContent;
+
+        let decodedDateText = decode(uploadDateText);
+
+        // "Вчера" means yesterday.
+        if (decodedDateText.includes("Вчера")) {
+          const yesterday = sub(new Date(), { days: 1 });
+          const formattedYesterday = format(yesterday, "d LLLL yyyy", {
+            locale: ru,
+          });
+          decodedDateText = decodedDateText.replace(
+            "Вчера", // "Change yesterday to the default expected date format"
+            formattedYesterday
+          );
+        }
+
+        const uploadDate = parse(
+          decodedDateText,
+          "d LLLL yyyy, HH:mm",
+          new Date(),
+          {
+            locale: ru,
+          }
+        );
+
+        const torrentButtons = Array.from(
+          gameDocument.querySelectorAll("a")
+        ).filter((a) => a.textContent?.includes("Torrent"));
+
+        const torrentPrePage = torrentButtons[0]?.getAttribute("href");
+        if (!torrentPrePage) return;
+
+        const torrentPage = await http
+          .get(torrentPrePage, {
+            encoding: "binary",
+            headers: {
+              Referer: gameLink,
+            },
+          })
+          .text();
+
+        const torrentDocument = new JSDOM(torrentPage).window.document;
+
+        const torrentLink = torrentDocument
+          .querySelector("a:nth-child(2)")
+          ?.getAttribute("href");
+
+        const torrentFile = Buffer.from(
+          await http
+            .get(`${torrentPrePage}/${torrentLink}`, {
+              responseType: "buffer",
+            })
+            .buffer()
+        );
+
+        const torrent = parseTorrent(torrentFile) as TorrentInstance;
+        const magnetLink = toMagnetURI({
+          infoHash: torrent.infoHash,
+        });
+
+        const torrentSizeInBytes = torrent.length;
+        const fileSizeFormatted =
+          torrentSizeInBytes >= 1024 ** 3
+            ? `${(torrentSizeInBytes / 1024 ** 3).toFixed(1)}GBs`
+            : `${(torrentSizeInBytes / 1024 ** 2).toFixed(1)}MBs`;
+
+        repacks.push({
+          fileSize: fileSizeFormatted,
+          magnet: magnetLink,
+          page: 1,
+          repacker: "onlinefix",
+          title: gameName,
+          uploadDate: uploadDate,
+        });
+      })
+    );
+  } catch (err) {
+    logger.error(err.message, { method: "getNewRepacksFromOnlineFix" });
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+  if (page === totalPages) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromOnlineFix(existingRepacks, page + 1, cookieJar);
+};