andrew.ng
/
SoldOut
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update scrap_new_items.js

This commit is contained in:
nguyentrungthat 2025-12-10 15:33:09 +07:00
parent eafe3a83ed
commit 45b29b0834
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scrap_new_items.js
View File

@ -36,7 +36,7 @@ function parseAndConvertToSydney(dateString, fromTimezone, formatDate) {
const dt = DateTime.fromFormat(fullDateString, `${format} yyyy`, { zone: fromTimezone }); const dt = DateTime.fromFormat(fullDateString, `${format} yyyy`, { zone: fromTimezone });
if (!dt.isValid) { if (!dt.isValid) {
console.log("❌ Invalid Luxon parse:", dt.invalidReason, fullDateString, format); // console.log("❌ Invalid Luxon parse:", dt.invalidReason, fullDateString, format);
return null; return null;
} }
@ -58,7 +58,7 @@ async function scrapeWithPuppeteer(store) {
let retries = 0; let retries = 0;
while (retries < 10) { while (retries < 10) {
try { try {
console.log(`Retry ${retries + 1}`); // console.log(`Retry ${retries + 1}`);
const html = await safeGetContent(page); const html = await safeGetContent(page);
// Detect Cloudflare or other blocking messages // Detect Cloudflare or other blocking messages
@ -284,7 +284,7 @@ async function scrapeWithPuppeteer(store) {
await browser.close(); await browser.close();
return results; return results;
} catch (err) { } catch (err) {
console.error("Error scrapeWithPuppeteer:", err); console.log("Error scrapeWithPuppeteer:", err);
return []; return [];
} }
} }
@ -335,7 +335,7 @@ async function main() {
database: process.env.MYSQL_DB_NAME, database: process.env.MYSQL_DB_NAME,
}); });
console.log("✅ Connected to MySQL"); // console.log("✅ Connected to MySQL");
// const errors = []; // const errors = [];
let configs = []; let configs = [];
@ -382,9 +382,9 @@ async function main() {
} }
}); });
} }
console.log(`Total configs to process: ${configs.length}`); // console.log(`Total configs to process: ${configs.length}`);
for (const store of configs) { for (const store of configs) {
console.log(`Processing ${store.data}`); // console.log(`Processing ${store.data}`);
// let items = await scrapeWithCheerio(store); // let items = await scrapeWithCheerio(store);
let items = await scrapeWithPuppeteer(store); let items = await scrapeWithPuppeteer(store);
for (const item of items) { for (const item of items) {
@ -406,17 +406,17 @@ async function main() {
} }
) )
.then((res) => { .then((res) => {
console.log(res.data, item.id, item.timeConvert, item.time); // console.log(res.data, item.id, item.timeConvert, item.time);
}) })
.catch((err) => { .catch((err) => {
console.error(err); console.log(err);
}); });
} }
} }
console.log("✅ Done scraping."); // console.log("✅ Done scraping.");
await db.end(); await db.end();
console.log("🔌 MySQL connection closed"); // console.log("🔌 MySQL connection closed");
process.exit(0); process.exit(0);
} }