external/hydra
1
0
Fork 0
mirror of https://github.com/hydralauncher/hydra.git synced 2025-03-09 15:40:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: crawl onlinefix games in parallel

Browse source
This commit is contained in:
João Martins 2024-04-12 01:30:42 -03:00
parent 47a2d82a90
commit 0b7db6cf13
No known key found for this signature in database
GPG key ID: 4F4EF2B738A71395
1 changed files with 89 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

174
src/main/services/repack-tracker/online-fix.ts
View file

@ -75,99 +75,103 @@ export const getNewRepacksFromOnlineFix = async (
const dom = new JSDOM(login);
const document = dom.window.document;
const repacks: GameRepackInput[] = [];
const articles = Array.from(document.querySelectorAll(".news"));
try {
const articles = Array.from(document.querySelectorAll(".news"));
await Promise.all(
articles.map(async (article) => {
const gameName = decode(
article.querySelector("h2.title")?.textContent?.trim()
)
.replace("по сети", "")
.trim();
for await (const article of articles) {
const gameName = decode(
article.querySelector("h2.title")?.textContent?.trim()
)
.replace("по сети", "")
.trim();
const gameLink = article.querySelector("a")?.getAttribute("href");
const gameLink = article.querySelector("a")?.getAttribute("href");
if (!gameLink) return;
if (!gameLink) continue;
const gamePage = await http
.get(gameLink, {
encoding: "binary",
})
.text();
const gameDom = new JSDOM(gamePage);
const gameDocument = gameDom.window.document;
const uploadDateText = gameDocument.querySelector(
"#dle-content > div > article > div.full-story-header.wide-block.clr > div.full-story-top-panel.clr > div.date.left > time"
).textContent;
let decodedDateText = decode(uploadDateText);
// "Вчера" significa ontem.
if (decodedDateText.includes("Вчера")) {
const yesterday = sub(new Date(), { days: 1 });
const formattedYesterday = format(yesterday, "d LLLL yyyy", {
locale: ru,
});
decodedDateText = decodedDateText.replace("Вчера", formattedYesterday);
}
const uploadDate = parse(
decodedDateText,
"d LLLL yyyy, HH:mm",
new Date(),
{
locale: ru,
}
);
const torrentButtons = Array.from(
gameDocument.querySelectorAll("a")
).filter((a) => a.textContent?.includes("Torrent"));
const torrentPrePage = torrentButtons[0]?.getAttribute("href");
if (!torrentPrePage) continue;
const torrentPage = await http
.get(torrentPrePage, {
encoding: "binary",
headers: {
Referer: gameLink,
},
})
.text();
const torrentDom = new JSDOM(torrentPage);
const torrentDocument = torrentDom.window.document;
const torrentLink = torrentDocument
.querySelector("a:nth-child(2)")
?.getAttribute("href");
const torrentFile = Buffer.from(
await http
.get(`${torrentPrePage}/${torrentLink}`, {
responseType: "buffer",
const gamePage = await http
.get(gameLink, {
encoding: "binary",
})
.buffer()
);
.text();
const torrent = parseTorrent(torrentFile);
const magnetLink = toMagnetURI({
infoHash: torrent.infoHash,
});
const gameDom = new JSDOM(gamePage);
const gameDocument = gameDom.window.document;
repacks.push({
fileSize: "NA",
magnet: magnetLink,
page: 1,
repacker: "onlinefix",
title: gameName,
uploadDate: uploadDate,
});
}
const uploadDateText = gameDocument.querySelector(
"#dle-content > div > article > div.full-story-header.wide-block.clr > div.full-story-top-panel.clr > div.date.left > time"
).textContent;
let decodedDateText = decode(uploadDateText);
// "Вчера" significa ontem.
if (decodedDateText.includes("Вчера")) {
const yesterday = sub(new Date(), { days: 1 });
const formattedYesterday = format(yesterday, "d LLLL yyyy", {
locale: ru,
});
decodedDateText = decodedDateText.replace(
"Вчера",
formattedYesterday
);
}
const uploadDate = parse(
decodedDateText,
"d LLLL yyyy, HH:mm",
new Date(),
{
locale: ru,
}
);
const torrentButtons = Array.from(
gameDocument.querySelectorAll("a")
).filter((a) => a.textContent?.includes("Torrent"));
const torrentPrePage = torrentButtons[0]?.getAttribute("href");
if (!torrentPrePage) return;
const torrentPage = await http
.get(torrentPrePage, {
encoding: "binary",
headers: {
Referer: gameLink,
},
})
.text();
const torrentDom = new JSDOM(torrentPage);
const torrentDocument = torrentDom.window.document;
const torrentLink = torrentDocument
.querySelector("a:nth-child(2)")
?.getAttribute("href");
const torrentFile = Buffer.from(
await http
.get(`${torrentPrePage}/${torrentLink}`, {
responseType: "buffer",
})
.buffer()
);
const torrent = parseTorrent(torrentFile);
const magnetLink = toMagnetURI({
infoHash: torrent.infoHash,
});
repacks.push({
fileSize: "NA",
magnet: magnetLink,
page: 1,
repacker: "onlinefix",
title: gameName,
uploadDate: uploadDate,
});
})
);
} catch (err) {
logger.error(err.message, { method: "getNewRepacksFromOnlineFix" });
}