andrew.ng
/
SoldOut
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update scrap_new_items.js

This commit is contained in:
nguyentrungthat 2025-12-10 15:33:09 +07:00
parent eafe3a83ed
commit 45b29b0834
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scrap_new_items.js
View File

@ -36,7 +36,7 @@ function parseAndConvertToSydney(dateString, fromTimezone, formatDate) {
const dt = DateTime.fromFormat(fullDateString, `${format} yyyy`, { zone: fromTimezone });
if (!dt.isValid) {
console.log("❌ Invalid Luxon parse:", dt.invalidReason, fullDateString, format);
// console.log("❌ Invalid Luxon parse:", dt.invalidReason, fullDateString, format);
return null;
}
@ -58,7 +58,7 @@ async function scrapeWithPuppeteer(store) {
let retries = 0;
while (retries < 10) {
try {
console.log(`Retry ${retries + 1}`);
// console.log(`Retry ${retries + 1}`);
const html = await safeGetContent(page);
// Detect Cloudflare or other blocking messages
@ -284,7 +284,7 @@ async function scrapeWithPuppeteer(store) {
await browser.close();
return results;
} catch (err) {
console.error("Error scrapeWithPuppeteer:", err);
console.log("Error scrapeWithPuppeteer:", err);
return [];
}
}
@ -335,7 +335,7 @@ async function main() {
database: process.env.MYSQL_DB_NAME,
});
console.log("✅ Connected to MySQL");
// console.log("✅ Connected to MySQL");
// const errors = [];
let configs = [];
@ -382,9 +382,9 @@ async function main() {
}
});
}
console.log(`Total configs to process: ${configs.length}`);
// console.log(`Total configs to process: ${configs.length}`);
for (const store of configs) {
console.log(`Processing ${store.data}`);
// console.log(`Processing ${store.data}`);
// let items = await scrapeWithCheerio(store);
let items = await scrapeWithPuppeteer(store);
for (const item of items) {
@ -406,17 +406,17 @@ async function main() {
}
)
.then((res) => {
console.log(res.data, item.id, item.timeConvert, item.time);
// console.log(res.data, item.id, item.timeConvert, item.time);
})
.catch((err) => {
console.error(err);
console.log(err);
});
}
}
console.log("✅ Done scraping.");
// console.log("✅ Done scraping.");
await db.end();
console.log("🔌 MySQL connection closed");
// console.log("🔌 MySQL connection closed");
process.exit(0);
}