Initial commit
This commit is contained in:
nguyentrungthat
commit
12b192780a
|
|
@ -0,0 +1,15 @@
|
|||
MAIL_MAILER=smtp
|
||||
MAIL_HOST=mail.apactech.io
|
||||
MAIL_PORT=465
|
||||
MAIL_USERNAME=admin@apactech.io
|
||||
MAIL_PASSWORD="BGK!dyt6upd2eax1bhz"
|
||||
MAIL_ENCRYPTION=ssl
|
||||
MAIL_FROM_ADDRESS=admin@apactech.io
|
||||
MAIL_FROM_NAME="${APP_NAME}"
|
||||
|
||||
DB_CONNECTION=mysql
|
||||
MYSQL_HOST=10.20.2.222
|
||||
MYSQL_PORT=3306
|
||||
MYSQL_DB_NAME=devnsw_disti
|
||||
MYSQL_USER=devnsw_devteam
|
||||
MYSQL_PASSWORD=12345678
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
node_modules/
|
||||
|
||||
.vscode/
|
||||
|
|
@ -0,0 +1,228 @@
|
|||
import "dotenv/config";
|
||||
import puppeteer from "puppeteer";
|
||||
import axios from "axios";
|
||||
import * as cheerio from "cheerio";
|
||||
import nodemailer from "nodemailer";
|
||||
import path from "path";
|
||||
import dayjs from "dayjs";
|
||||
import mysql from "mysql2/promise";
|
||||
|
||||
const EMAILS = ["andrew.ng@apactech.io"];
|
||||
|
||||
const LIST_STORE = [
|
||||
{
|
||||
name: "ocdepot",
|
||||
url: "https://www.ebay.com/sch/i.html?_dkr=1&iconV2Request=true&_blrs=recall_filtering&_ssn=ocdepot&store_name=ocdepot&_oac=1&LH_Sold=1&rt=nc&_ipg=240",
|
||||
},
|
||||
{
|
||||
name: "itinstock",
|
||||
url: "https://www.ebay.com/sch/i.html?_dkr=1&iconV2Request=true&_blrs=recall_filtering&_ssn=itinstock&store_name=itinstock&_oac=1&LH_Sold=1&rt=nc&_ipg=240",
|
||||
},
|
||||
{
|
||||
name: "kartechllc",
|
||||
url: "https://www.ebay.com/sch/i.html?_dkr=1&iconV2Request=true&_blrs=recall_filtering&_ssn=luckywolf29&store_name=kartechllc&_oac=1&LH_Sold=1&rt=nc&_ipg=240",
|
||||
},
|
||||
{
|
||||
name: "g-electronic",
|
||||
url: "https://www.ebay.com/sch/i.html?_dkr=1&iconV2Request=true&_blrs=recall_filtering&_ssn=g-electronic&store_name=gelectronic&_oac=1&LH_Sold=1&rt=nc&_ipg=240",
|
||||
},
|
||||
];
|
||||
|
||||
async function scrapeWithPuppeteer(url) {
|
||||
try {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ["--no-sandbox", "--disable-setuid-sandbox"],
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
await page.goto(url, { waitUntil: "networkidle2" });
|
||||
|
||||
// Wait for cards or detect Cloudflare
|
||||
const html = await page.content();
|
||||
const needBrowserCheck = html.includes("Checking your browser");
|
||||
if (needBrowserCheck) {
|
||||
await page.waitForSelector("li.s-card--horizontal", { timeout: 15000 }).catch(() => null);
|
||||
}
|
||||
|
||||
const items = await page.$$eval("li.s-card--horizontal", (nodes) => {
|
||||
const results = [];
|
||||
nodes.forEach((node) => {
|
||||
const payload = {};
|
||||
const soldEl = node.querySelector(".s-card__caption .su-styled-text.positive.default");
|
||||
if (!soldEl) return;
|
||||
const soldText = soldEl.textContent.trim().replace("Sold", "").trim();
|
||||
payload.sold_out_date = soldText;
|
||||
|
||||
const date = new Date(soldText);
|
||||
if (isNaN(date.getTime())) return;
|
||||
const daysDiff = (Date.now() - date.getTime()) / (1000 * 60 * 60 * 24);
|
||||
if (daysDiff > 7) return;
|
||||
|
||||
const linkEl = node.querySelector("div.su-media__image a");
|
||||
if (linkEl) {
|
||||
payload.link_detail = linkEl.href;
|
||||
const match = linkEl.href.match(/\/itm\/(\d+)/);
|
||||
if (match) payload.id = match[1];
|
||||
}
|
||||
|
||||
const titleEl = node.querySelector(".s-card__title");
|
||||
if (titleEl) {
|
||||
payload.name = titleEl.textContent.replace(/New\s*listing/i, "").trim();
|
||||
payload.description = payload.name;
|
||||
}
|
||||
|
||||
const conditionEl = node.querySelector(".s-card__subtitle");
|
||||
if (conditionEl) payload.condition = conditionEl.textContent.trim();
|
||||
|
||||
const priceEl = node.querySelector(".s-card__price");
|
||||
if (priceEl) {
|
||||
let txt = priceEl.textContent.replace(",", "").trim();
|
||||
txt = txt.replace("£", "GBP ").replace("$", "USD ");
|
||||
const match = txt.match(/([A-Za-z]{3})?\s?([\d.,]+)\s?([A-Za-z]{3})?/);
|
||||
if (match) {
|
||||
payload.currency = match[1] || "";
|
||||
payload.price = match[2] || "";
|
||||
}
|
||||
}
|
||||
|
||||
if (payload.id) results.push(payload);
|
||||
});
|
||||
return results;
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
return items;
|
||||
} catch (error) {
|
||||
console.error("Puppeteer scraping failed:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeWithCheerio(url) {
|
||||
const { data } = await axios.get(url);
|
||||
if (data.includes("Checking your browser")) return null;
|
||||
const $ = cheerio.load(data);
|
||||
const items = [];
|
||||
$("li.s-card--horizontal").each((i, el) => {
|
||||
const payload = {};
|
||||
const soldText = $(el).find(".s-card__caption .su-styled-text.positive.default").text().trim();
|
||||
if (!soldText) return;
|
||||
payload.sold_out_date = soldText.replace("Sold", "").trim();
|
||||
const date = new Date(soldText.replace("Sold", "").trim());
|
||||
const daysDiff = (Date.now() - date.getTime()) / (1000 * 60 * 60 * 24);
|
||||
if (daysDiff > 7) return;
|
||||
|
||||
const linkEl = $(el).find("div.su-media__image a").attr("href");
|
||||
if (linkEl) {
|
||||
payload.link_detail = linkEl;
|
||||
const match = linkEl.match(/\/itm\/(\d+)/);
|
||||
if (match) payload.id = match[1];
|
||||
}
|
||||
|
||||
payload.name = $(el)
|
||||
.find(".s-card__title")
|
||||
.text()
|
||||
.replace(/New\s*listing/i, "")
|
||||
.trim();
|
||||
payload.condition = $(el).find(".s-card__subtitle").text().trim();
|
||||
const priceText = $(el).find(".s-card__price").text().trim().replace(",", "");
|
||||
const match = priceText.match(/([£$€A-Za-z]{1,5})?\s?([\d.,]+)\s?([£$€A-Za-z]{1,5})?/);
|
||||
if (match) {
|
||||
payload.price = match[2];
|
||||
payload.currency = match[1] || match[3];
|
||||
}
|
||||
|
||||
if (payload.id) items.push(payload);
|
||||
});
|
||||
return items;
|
||||
}
|
||||
|
||||
async function sendMail(subject, body, attachmentPath = null) {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: process.env.MAIL_HOST,
|
||||
port: process.env.MAIL_PORT,
|
||||
secure: true,
|
||||
auth: { user: process.env.MAIL_USERNAME, pass: process.env.MAIL_PASSWORD },
|
||||
});
|
||||
|
||||
const mailOptions = {
|
||||
from: process.env.MAIL_USERNAME,
|
||||
to: EMAILS.join(","),
|
||||
subject,
|
||||
text: body,
|
||||
};
|
||||
|
||||
if (attachmentPath) {
|
||||
mailOptions.attachments = [{ filename: path.basename(attachmentPath), path: attachmentPath }];
|
||||
}
|
||||
|
||||
await transporter.sendMail(mailOptions);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// 1️⃣ Connect to MySQL
|
||||
const db = await mysql.createConnection({
|
||||
host: process.env.MYSQL_HOST, // e.g. '127.0.0.1' or remote IP
|
||||
user: process.env.MYSQL_USER,
|
||||
password: process.env.MYSQL_PASSWORD,
|
||||
database: process.env.MYSQL_DB_NAME,
|
||||
});
|
||||
|
||||
console.log("✅ Connected to MySQL");
|
||||
|
||||
const errors = [];
|
||||
const inserted = [];
|
||||
|
||||
for (const store of LIST_STORE) {
|
||||
console.log(`Processing ${store.name}`);
|
||||
let items = await scrapeWithCheerio(store.url);
|
||||
if (!items) items = await scrapeWithPuppeteer(store.url);
|
||||
|
||||
let count = 0;
|
||||
for (const item of items) {
|
||||
// 2️⃣ Check if record exists
|
||||
const [rows] = await db.execute("SELECT id FROM items_sold_out WHERE id = ?", [item.id]);
|
||||
if (rows.length > 0) continue;
|
||||
|
||||
// 3️⃣ Insert new record
|
||||
const priceText = item.price || "";
|
||||
const amount = parseFloat(priceText.replace(/[^\d.]/g, "").replace(/,/g, ""));
|
||||
await db.execute(
|
||||
`INSERT INTO items_sold_out (id, name, \`condition\`, price, currency, link_detail, shop_name, sold_out_date, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
item.id,
|
||||
item.name,
|
||||
item.condition,
|
||||
amount || 0,
|
||||
item.currency,
|
||||
item.link_detail,
|
||||
store.name,
|
||||
item.sold_out_date,
|
||||
dayjs().format("YYYY-MM-DD HH:mm:ss"),
|
||||
dayjs().format("YYYY-MM-DD HH:mm:ss"),
|
||||
]
|
||||
);
|
||||
count++;
|
||||
}
|
||||
|
||||
if (count === 0) errors.push({ url: store.url, message: "No new items inserted" });
|
||||
inserted.push({ name: store.name, count });
|
||||
}
|
||||
|
||||
// 4️⃣ Send email report
|
||||
if (errors.length > 3) {
|
||||
const msg = errors.map((e) => `URL: ${e.url}\nMessage: ${e.message}`).join("\n\n");
|
||||
await sendMail("[New Items] - Scraping Sold Out Error Report", msg);
|
||||
} else {
|
||||
const msg = inserted.map((i) => `Shop: <b>${i.name}</b>\nInserted: <b>${i.count}</b>`).join("\n\n");
|
||||
await sendMail("[New Items] - Scraping Sold Out Success", msg);
|
||||
}
|
||||
|
||||
console.log("✅ Done scraping.");
|
||||
|
||||
// 5️⃣ Close MySQL connection
|
||||
await db.end();
|
||||
console.log("🔌 MySQL connection closed");
|
||||
}
|
||||
|
||||
main().catch((err) => console.error(err));
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"name": "scrapsoldout",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"axios": "^1.12.2",
|
||||
"cheerio": "^1.1.0",
|
||||
"dayjs": "^1.11.18",
|
||||
"dotenv": "^17.2.3",
|
||||
"mysql2": "^3.15.2",
|
||||
"nodemailer": "^7.0.9",
|
||||
"puppeteer": "^24.24.1",
|
||||
"sqlite": "^5.1.1",
|
||||
"sqlite3": "^5.1.7",
|
||||
"xlsx": "^0.18.5"
|
||||
},
|
||||
"type": "module"
|
||||
}
|
||||
Loading…
Reference in New Issue