46 lines
1.3 KiB
JavaScript
46 lines
1.3 KiB
JavaScript
import dotenv from "dotenv";
|
|
import "dotenv/config";
|
|
dotenv.config();
|
|
import { getScapeConfigs, upsertScapeItems } from "./apis/scrape.js";
|
|
import { ScrapConfigsService } from "./services/scrap-configs-service.js";
|
|
import browser from "./system/browser.js";
|
|
|
|
const init = async () => {
|
|
const scrapConfigs = await getScapeConfigs();
|
|
|
|
try {
|
|
const page = await browser.newPage();
|
|
|
|
await page.setUserAgent(
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
|
|
);
|
|
|
|
const models = ScrapConfigsService.scrapModels(scrapConfigs, page);
|
|
console.log(`Loaded ${models.length} scrape models`);
|
|
|
|
for (let model of models) {
|
|
try {
|
|
await model.action();
|
|
|
|
console.log(`Loaded model [${model?.web_bid?.origin_url || ""}]`);
|
|
for (const key of Object.keys(model.results)) {
|
|
const dataArray = model.results[key];
|
|
const result = await upsertScapeItems(dataArray);
|
|
console.log(`Upserted ${dataArray.length} items for ${key}:`, result);
|
|
}
|
|
} catch (err) {
|
|
console.error(
|
|
`Error in model ${model.config?.name || "unknown"}:`,
|
|
err
|
|
);
|
|
}
|
|
}
|
|
|
|
await page.close();
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
};
|
|
|
|
init();
|