external/hydra
1
0
Fork 0
mirror of https://github.com/hydralauncher/hydra.git synced 2025-03-09 15:40:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: crawl onlinefix games in parallel

Browse source
This commit is contained in:
João Martins 2024-04-12 01:30:42 -03:00
parent 47a2d82a90
commit 0b7db6cf13
No known key found for this signature in database
GPG key ID: 4F4EF2B738A71395
1 changed files with 89 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/main/services/repack-tracker/online-fix.ts
View file

@ -75,11 +75,11 @@ export const getNewRepacksFromOnlineFix = async (
const dom = new JSDOM(login); const dom = new JSDOM(login);
const document = dom.window.document; const document = dom.window.document;
const repacks: GameRepackInput[] = []; const repacks: GameRepackInput[] = [];
try {
const articles = Array.from(document.querySelectorAll(".news")); const articles = Array.from(document.querySelectorAll(".news"));
for await (const article of articles) { try {
await Promise.all(
articles.map(async (article) => {
const gameName = decode( const gameName = decode(
article.querySelector("h2.title")?.textContent?.trim() article.querySelector("h2.title")?.textContent?.trim()
) )
@ -88,7 +88,7 @@ export const getNewRepacksFromOnlineFix = async (
const gameLink = article.querySelector("a")?.getAttribute("href"); const gameLink = article.querySelector("a")?.getAttribute("href");
if (!gameLink) continue; if (!gameLink) return;
const gamePage = await http const gamePage = await http
.get(gameLink, { .get(gameLink, {
@ -111,7 +111,10 @@ export const getNewRepacksFromOnlineFix = async (
const formattedYesterday = format(yesterday, "d LLLL yyyy", { const formattedYesterday = format(yesterday, "d LLLL yyyy", {
locale: ru, locale: ru,
}); });
decodedDateText = decodedDateText.replace("Вчера", formattedYesterday); decodedDateText = decodedDateText.replace(
"Вчера",
formattedYesterday
);
} }
const uploadDate = parse( const uploadDate = parse(
@ -128,7 +131,7 @@ export const getNewRepacksFromOnlineFix = async (
).filter((a) => a.textContent?.includes("Torrent")); ).filter((a) => a.textContent?.includes("Torrent"));
const torrentPrePage = torrentButtons[0]?.getAttribute("href"); const torrentPrePage = torrentButtons[0]?.getAttribute("href");
if (!torrentPrePage) continue; if (!torrentPrePage) return;
const torrentPage = await http const torrentPage = await http
.get(torrentPrePage, { .get(torrentPrePage, {
@ -167,7 +170,8 @@ export const getNewRepacksFromOnlineFix = async (
title: gameName, title: gameName,
uploadDate: uploadDate, uploadDate: uploadDate,
}); });
} })
);
} catch (err) { } catch (err) {
logger.error(err.message, { method: "getNewRepacksFromOnlineFix" }); logger.error(err.message, { method: "getNewRepacksFromOnlineFix" });
} }