ebayDeepScan/index.js

556 lines
21 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

require('dotenv').config();
const axios = require('axios');
const XLSX = require('xlsx');
const fs = require('fs');
const path = require('path');
// ─── CONFIG ──────────────────────────────────────────────────────────────────
const CFG = {
ebayClientId: process.env.EBAY_CLIENT_ID,
ebayClientSecret: process.env.EBAY_CLIENT_SECRET,
isSandbox: (process.env.EBAY_ENVIRONMENT || 'production') === 'sandbox',
marketplace: process.env.EBAY_MARKETPLACE || 'EBAY_US',
shipZip: process.env.SHIP_TO_ZIP || '10001',
shipCountry: process.env.SHIP_TO_COUNTRY || 'US',
itemsPerPage: parseInt(process.env.ITEMS_PER_PAGE || '50'),
priceSheet: process.env.PRICE_SHEET || './prices.xlsx',
outputDir: process.env.OUTPUT_DIR || './output',
priceRatio: parseFloat(process.env.PRICE_RATIO || '0.85'),
rowBatchSize: parseInt(process.env.ROW_BATCH_SIZE || '5'),
pageSize: parseInt(process.env.PAGE_SIZE || '200'),
maxItemsPerQuery: parseInt(process.env.MAX_ITEMS_PER_QUERY || '500'),
};
const EBAY_SITES = [
{ id: 'US', market: 'EBAY_US', zip: '10001', country: 'US', domain: 'ebay.com' },
{ id: 'UK', market: 'EBAY_GB', zip: 'W1A 1AA', country: 'GB', domain: 'ebay.co.uk' },
{ id: 'AU', market: 'EBAY_AU', zip: '2000', country: 'AU', domain: 'ebay.com.au' },
{ id: 'CA', market: 'EBAY_CA', zip: 'M5H 2N2', country: 'CA', domain: 'ebay.ca' },
{ id: 'DE', market: 'EBAY_DE', zip: '10115', country: 'DE', domain: 'ebay.de' },
];
const BASE_URL = CFG.isSandbox ? 'api.sandbox.ebay.com' : 'api.ebay.com';
// ─── LOGGER ──────────────────────────────────────────────────────────────────
const startTime = Date.now();
const logLines = [];
function log(msg, level = 'INFO') {
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
const line = `[${elapsed}s] [${level}] ${msg}`;
console.log(line);
logLines.push(line);
}
function logSection(title) {
const bar = '─'.repeat(60);
log('');
log(bar);
log(` ${title}`);
log(bar);
}
// ─── UTIL ─────────────────────────────────────────────────────────────────────
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
function ensureDir(dir) {
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}
// ─── DEFECTIVE LISTING FILTER ─────────────────────────────────────────────────
// Patterns that indicate a listing is broken / for-parts / non-functional
const DEFECTIVE_PATTERNS = [
/\bnon[\s-]?working\b/i,
/\bnot[\s-]?working\b/i,
/\bno[\s-]?working\b/i,
/\bfor[\s-]?parts?\b/i,
/\bparts?[\s-]?only\b/i,
/\bparts?[\s-]?or[\s-]?repair\b/i,
/\bbroken\b/i,
/\bdefective\b/i,
/\bfaulty\b/i,
/\bdead[\s-]?on[\s-]?arrival\b/i,
/\bdoa\b/i,
/\bas[\s-]?is\b/i,
/\bdamaged\b/i,
/\bmalfunctioning\b/i,
/\bno[\s-]?post\b/i,
];
function isDefectiveListing(item) {
const title = (item.title || '').toLowerCase();
return DEFECTIVE_PATTERNS.some(re => re.test(title));
}
// ─── LOAD PRICE SHEET ────────────────────────────────────────────────────────
/**
* Reads prices.xlsx and returns an array of row objects.
* Expected columns (row 0 = header):
* Specs | Manufacturer | Part Number | Capacity (GB) | Rank |
* Speed (MT/s) | Speed Grade | Type | Price
*/
function loadPriceSheet() {
const wb = XLSX.readFile(CFG.priceSheet);
const ws = wb.Sheets[wb.SheetNames[0]];
const rows = XLSX.utils.sheet_to_json(ws, { header: 1 });
const headers = rows[0].map(h => String(h).trim());
const data = [];
for (let i = 1; i < rows.length; i++) {
const row = rows[i];
if (!row || !row.length) continue;
const obj = {};
headers.forEach((h, idx) => { obj[h] = row[idx] !== undefined ? row[idx] : ''; });
const partNumber = String(obj['Part Number'] || '').trim();
const price = parseFloat(obj['Price']);
if (!partNumber || isNaN(price)) continue;
data.push({
specs: String(obj['Specs'] || '').trim(),
manufacturer: String(obj['Manufacturer'] || '').trim(),
partNumber,
capacity: obj['Capacity (GB)'],
rank: String(obj['Rank'] || '').trim(),
speedMTs: obj['Speed (MT/s)'],
speedGrade: String(obj['Speed Grade'] || '').trim(),
type: String(obj['Type'] || '').trim(),
price,
});
}
log(`Loaded ${data.length} rows from ${CFG.priceSheet}`);
return data;
}
// ─── BUILD SEARCH QUERIES ────────────────────────────────────────────────────
/**
* Returns 3 search query strings for a price sheet row:
* 1. Part Number only
* 2. Full spec with Speed (MT/s): e.g. "Samsung 64GB 2Rx4 4800 RDIMM"
* 3. Full spec with Speed Grade: e.g. "Samsung 64GB 2Rx4 PC5-38400 RDIMM"
*/
function buildQueries(row) {
const { manufacturer, partNumber, capacity, rank, speedMTs, speedGrade, type } = row;
const capStr = capacity ? `${capacity}GB` : '';
const speedStr = speedMTs ? String(speedMTs) : '';
const q1 = partNumber;
// e.g. "Samsung 64GB 2Rx4 4800 RDIMM"
const q2Parts = [manufacturer, capStr, rank, speedStr, type].filter(Boolean);
const q2 = q2Parts.join(' ');
// e.g. "Samsung 64GB 2Rx4 PC5-38400 RDIMM"
const q3Parts = [manufacturer, capStr, rank, speedGrade?.split("-")[1], type].filter(Boolean);
const q3 = q3Parts.join(' ');
return [
{ label: 'PartNumber', query: q1 },
{ label: 'SpeedMTs', query: q2 },
{ label: 'SpeedGrade', query: q3 },
].filter(q => q.query.trim().length > 0);
}
// ─── EBAY AUTH ───────────────────────────────────────────────────────────────
async function getEbayToken() {
const creds = Buffer.from(`${CFG.ebayClientId}:${CFG.ebayClientSecret}`).toString('base64');
const res = await axios.post(
`https://${BASE_URL}/identity/v1/oauth2/token`,
'grant_type=client_credentials&scope=https://api.ebay.com/oauth/api_scope',
{ headers: { Authorization: `Basic ${creds}`, 'Content-Type': 'application/x-www-form-urlencoded' } }
);
return res.data.access_token;
}
// ─── EBAY SEARCH (paginated) ─────────────────────────────────────────────────
// eBay US category ID for Computer Memory — scoping here forces eBay's engine
// to return only memory listings that actually match the searched keywords.
// const MEMORY_CATEGORY_ID = '170083';
/**
* Paginated eBay search.
* Scoped by the specific site marketplace and location context.
*/
async function searchEbay(token, keyword, site) {
const headers = {
Authorization: `Bearer ${token}`,
'X-EBAY-C-MARKETPLACE-ID': site.market,
'X-EBAY-C-ENDUSERCTX': `zip=${site.zip},country=${site.country}`,
};
const pageSize = Math.min(CFG.pageSize, 200);
const cap = CFG.maxItemsPerQuery;
// Convert "Micron 64GB 4DRx4 2933 LRDIMM" → '"Micron" "64GB" "4DRx4" "2933" "LRDIMM"'
// This explicitly forces eBay Browse API to treat each word as an exact match term.
const exactQuery = keyword.trim().split(/\s+/).map(w => `"${w}"`).join(' ');
const baseUrl = `https://${BASE_URL}/buy/browse/v1/item_summary/search`
+ `?q=${encodeURIComponent(exactQuery)}`
// + `&category_ids=${MEMORY_CATEGORY_ID}`
+ `&fieldgroups=EXTENDED`
+ `&filter=conditionIds:{1000|1500|2000|2500|3000|4000|5000|6000|7000}`;
const all = [];
let offset = 0;
let total = null;
do {
const url = `${baseUrl}&limit=${pageSize}&offset=${offset}`;
log(` [API] GET ${url}`, 'DEBUG');
try {
const res = await axios.get(url, { headers });
const data = res.data;
if (total === null) total = data.total || 0;
const page = data.itemSummaries || [];
all.push(...page);
offset += pageSize;
if (!page.length || all.length >= cap || offset >= total) break;
await sleep(150); // polite inter-page delay
} catch (err) {
const msg = err.response?.data?.errors?.[0]?.message || err.message;
log(` eBay search error for "${keyword}" offset=${offset}: ${msg}`, 'WARN');
break;
}
} while (true);
return all.slice(0, cap);
}
// ─── PRICE EXTRACTION ────────────────────────────────────────────────────────
function extractPrice(item) {
const p = item.price || item.currentBidPrice || {};
return parseFloat(p.value || 0);
}
function extractShipping(item) {
const opts = item.shippingOptions || [];
if (!opts.length) return { cost: 0, label: 'N/A' };
const first = opts[0];
if (first.shippingCostType === 'FREE') return { cost: 0, label: 'Free' };
return { cost: parseFloat(first.shippingCost?.value || 0), label: first.shippingCostType || '' };
}
/**
* Detect quantity in listing (lot size / pack quantity).
* Checks availableQuantity first, then title patterns.
*/
function extractQty(item) {
const title = item.title || '';
const patterns = [
/\bLOT\s+OF\s+(\d+)\b/i,
/\b(\d+)\s*(?:PACK|PCS?|PIECES?|UNITS?)\b/i,
/\bQTY\s*[:\-]?\s*(\d+)\b/i,
/\b(\d+)\s*-\s*PACK\b/i,
/^(\d+)x\s/i,
/\bX(\d+)\b/i,
];
for (const p of patterns) {
const m = title.match(p);
if (m && parseInt(m[1]) > 1) return parseInt(m[1]);
}
return 1;
}
// ─── COMPARE ─────────────────────────────────────────────────────────────────
/**
* For a given eBay item and row, compute avg price per unit and compare with target.
*/
function compareItem(item, row, searchLabel, site) {
const rawPrice = extractPrice(item);
const { cost: shipCost, label: shipLabel } = extractShipping(item);
const qty = extractQty(item);
const total = parseFloat((rawPrice + shipCost).toFixed(2));
const avg = qty > 1 ? parseFloat((total / qty).toFixed(2)) : total;
const threshold = parseFloat((row.price * CFG.priceRatio).toFixed(2));
const pass = avg <= threshold;
const available = item.availableQuantity ?? 1;
const itemId = (item.itemWebUrl || '').match(/\/itm\/(\d+)/)?.[1] || item.itemId;
const location = item.itemLocation?.country || 'Unknown';
// Force the domain to the prioritized site domain so the link matches the search marketplace
const finalUrl = `https://www.${site.domain}/itm/${itemId}`;
return {
'Search Type': searchLabel,
'Part Number': row.partNumber,
'Manufacturer': row.manufacturer,
'Specs': row.specs,
'eBay Title': item.title,
'Item ID': itemId,
'Item Location': location,
'eBay URL': finalUrl,
'eBay Price (USD)': rawPrice,
'Shipping (USD)': shipCost,
'Shipping Label': shipLabel,
'Total (USD)': total,
'Qty in Listing': qty,
'Avg Price/Unit': avg,
'Available Qty': available,
'Target Price': row.price,
[`Target × ${CFG.priceRatio}`]: threshold,
'Pass/Fail': pass ? 'PASS' : 'FAIL',
'Profit (est.)': pass ? parseFloat(((row.price - avg) * available).toFixed(2)) : null,
};
}
// ─── MAIN SEARCH LOOP ────────────────────────────────────────────────────────
/**
* Process a single price-sheet row: run 3 eBay queries across 5 sites natively,
* filter defective and CN listings, dedupe globally by priority, and build results map.
*/
async function processRow(token, row, rowIndex, totalRows, globalResultsMap) {
const queries = buildQueries(row);
log(`[${rowIndex + 1}/${totalRows}] ${row.partNumber}${row.manufacturer} ${row.specs}`);
for (const site of EBAY_SITES) {
for (const { label, query } of queries) {
log(` [${rowIndex + 1}] [${site.id}] [${label}] Searching: "${query}"`);
const items = await searchEbay(token, query, site);
let skipped = 0, cnSkipped = 0;
for (const item of items) {
const itemId = (item.itemWebUrl || '').match(/\/itm\/(\d+)/)?.[1] || item.itemId;
if (!itemId) continue;
// Skip broken / non-working / for-parts listings
if (isDefectiveListing(item)) { skipped++; continue; }
// Skip China listings
const locationCountry = item.itemLocation?.country;
if (locationCountry === 'CN') { cnSkipped++; continue; }
// Determine priority: 2 if item natively located in the searched site, 1 otherwise
const priority = (locationCountry === site.country) ? 2 : 1;
// Dedupe globally with priority overwrite
const existing = globalResultsMap.get(itemId);
if (!existing || existing._priority < priority) {
const processedRow = compareItem(item, row, label, site);
processedRow._priority = priority; // tag internal state
globalResultsMap.set(itemId, processedRow);
}
}
const notes = [];
if (skipped) notes.push(`${skipped} defective`);
if (cnSkipped) notes.push(`${cnSkipped} CN-located`);
log(` [${rowIndex + 1}] [${site.id}] [${label}] → ${items.length} raw results${notes.length ? `, skipped: ${notes.join(', ')}` : ''}`);
await sleep(CFG.requestDelay);
}
}
}
/**
* Run all rows in concurrent batches of CFG.rowBatchSize.
*/
async function runSearch(token, rows) {
logSection(`SEARCHING EBAY — ${rows.length} rows, batch size ${CFG.rowBatchSize} across ${EBAY_SITES.length} sites`);
const batchSize = CFG.rowBatchSize;
const totalBatch = Math.ceil(rows.length / batchSize);
// We use a Map to keep track of the absolute best-priority matching row for each itemId globally
const globalResultsMap = new Map();
for (let b = 0; b < rows.length; b += batchSize) {
const batch = rows.slice(b, b + batchSize);
const batchNum = Math.floor(b / batchSize) + 1;
log(`\n── Batch ${batchNum}/${totalBatch} (rows ${b + 1}${Math.min(b + batchSize, rows.length)}) ──`);
await Promise.all(
batch.map((row, idx) => processRow(token, row, b + idx, rows.length, globalResultsMap))
);
}
// Convert Map values to flat array and remove internal _priority field
return Array.from(globalResultsMap.values()).map(r => {
delete r._priority;
return r;
});
}
// ─── ITEM DETAIL API ─────────────────────────────────────────────────────
/**
* Fetch real available quantity from the item detail endpoint.
* Returns null on error.
*/
async function fetchAvailableQty(token, itemId) {
const headers = {
Authorization: `Bearer ${token}`,
'X-EBAY-C-MARKETPLACE-ID': CFG.marketplace,
'X-EBAY-C-ENDUSERCTX': `zip=${CFG.shipZip},country=${CFG.shipCountry}`,
};
try {
const res = await axios.get(
`https://${BASE_URL}/buy/browse/v1/item/v1|${itemId}|0`,
{ headers }
);
const data = res.data;
// Prefer estimatedAvailabilities, fall back to quantity fields
const est = data.estimatedAvailabilities?.[0];
if (est?.estimatedAvailableQuantity != null) return est.estimatedAvailableQuantity;
if (data.quantity != null) return data.quantity;
return null;
} catch {
return null;
}
}
/**
* For every PASS row, concurrently fetch the real Available Qty from eBay
* item detail API and update the row in-place.
* Runs in batches of 5 to avoid hammering the API.
*/
async function enrichPassItems(token, results) {
const passRows = results.filter(r => r['Pass/Fail'] === 'PASS');
if (!passRows.length) { log('No PASS items to enrich.'); return; }
logSection(`ENRICHING ${passRows.length} PASS ITEMS — fetching Available Qty`);
const batchSize = 5;
for (let b = 0; b < passRows.length; b += batchSize) {
const batch = passRows.slice(b, b + batchSize);
await Promise.all(batch.map(async row => {
const itemId = String(row['Item ID']);
const qty = await fetchAvailableQty(token, itemId);
if (qty !== null) {
row['Available Qty'] = qty;
// Recompute profit with real qty
const avg = row['Avg Price/Unit'];
const target = row['Target Price'];
row['Profit (est.)'] = parseFloat(((target - avg) * qty).toFixed(2));
}
log(` Item ${itemId} → Available Qty: ${qty ?? '(failed)'}`);
}));
if (b + batchSize < passRows.length) await sleep(200);
}
}
// ─── OUTPUT ───────────────────────────────────────────────────────────────────
function applyHyperlinks(ws, headerName, totalRows) {
const range = XLSX.utils.decode_range(ws['!ref'] || 'A1');
let urlCol = -1;
for (let c = range.s.c; c <= range.e.c; c++) {
if (ws[XLSX.utils.encode_cell({ r: 0, c })]?.v === headerName) { urlCol = c; break; }
}
if (urlCol < 0) return;
for (let r = 1; r <= totalRows; r++) {
const addr = XLSX.utils.encode_cell({ r, c: urlCol });
const cell = ws[addr];
if (!cell?.v || !String(cell.v).startsWith('http')) continue;
const url = String(cell.v);
delete cell.f; delete cell.z;
cell.t = 's'; cell.v = 'View on eBay';
cell.l = { Target: url, Tooltip: url };
}
}
function saveOutputs(rows, runId) {
ensureDir(CFG.outputDir);
// Sort: PASS first → profit desc → avg price asc
rows.sort((a, b) => {
if (a['Pass/Fail'] !== b['Pass/Fail']) return a['Pass/Fail'] === 'PASS' ? -1 : 1;
const profitA = a['Profit (est.)'] ?? -Infinity;
const profitB = b['Profit (est.)'] ?? -Infinity;
if (profitA !== profitB) return profitB - profitA;
return (a['Avg Price/Unit'] || 0) - (b['Avg Price/Unit'] || 0);
});
const passCount = rows.filter(r => r['Pass/Fail'] === 'PASS').length;
log(`Total rows : ${rows.length}`);
log(`PASS : ${passCount}`);
log(`FAIL : ${rows.length - passCount}`);
// Excel
const xlsxPath = path.join(CFG.outputDir, `results_${runId}.xlsx`);
const wb = XLSX.utils.book_new();
const ws = XLSX.utils.json_to_sheet(rows);
applyHyperlinks(ws, 'eBay URL', rows.length);
ws['!cols'] = [
{ wch: 12 }, // Search Type
{ wch: 22 }, // Part Number
{ wch: 14 }, // Manufacturer
{ wch: 24 }, // Specs
{ wch: 50 }, // eBay Title
{ wch: 14 }, // Item ID
{ wch: 14 }, // Item Location
{ wch: 16 }, // eBay URL
{ wch: 15 }, // eBay Price
{ wch: 13 }, // Shipping
{ wch: 13 }, // Shipping Label
{ wch: 13 }, // Total
{ wch: 12 }, // Qty
{ wch: 14 }, // Avg Price/Unit
{ wch: 12 }, // Available
{ wch: 13 }, // Target Price
{ wch: 14 }, // Target × ratio
{ wch: 10 }, // Pass/Fail
{ wch: 13 }, // Profit
];
XLSX.utils.book_append_sheet(wb, ws, 'Results');
XLSX.writeFile(wb, xlsxPath);
log(`Excel saved : ${xlsxPath}`);
// JSON
const jsonPath = path.join(CFG.outputDir, `results_${runId}.json`);
fs.writeFileSync(jsonPath, JSON.stringify(rows, null, 2));
log(`JSON saved : ${jsonPath}`);
// Log file
const logPath = path.join(CFG.outputDir, `run_${runId}.log`);
fs.writeFileSync(logPath, logLines.join('\n'));
log(`Log saved : ${logPath}`);
return { xlsxPath, jsonPath, logPath };
}
// ─── MAIN ────────────────────────────────────────────────────────────────────
async function main() {
const runId = Date.now();
logSection(`eBay Price Scanner — Run ${runId}`);
// Validate required env
const missing = ['EBAY_CLIENT_ID', 'EBAY_CLIENT_SECRET'].filter(k => !process.env[k]);
if (missing.length) {
log(`Missing env vars: ${missing.join(', ')}`, 'ERROR');
process.exit(1);
}
// Load price sheet
const priceRows = loadPriceSheet();
if (!priceRows.length) {
log('No valid rows found in price sheet. Exiting.', 'ERROR');
process.exit(1);
}
// Get eBay token
log('Getting eBay token...');
const token = await getEbayToken();
log('eBay token acquired');
// Search + compare
const results = await runSearch(token, priceRows);
// Enrich PASS items with real Available Qty from detail API
await enrichPassItems(token, results);
// Save outputs
logSection('SAVING OUTPUTS');
saveOutputs(results, runId);
logSection('DONE');
log(`Total elapsed : ${((Date.now() - startTime) / 1000).toFixed(1)}s`);
log(`Run ID : ${runId}`);
}
main().catch(err => {
log(`FATAL: ${err.stack || err.message}`, 'ERROR');
process.exit(1);
});