zelda
/
bid-tool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extract content ram product to a new mail

This commit is contained in:
Admin 2026-01-30 11:03:52 +07:00
parent f4b677e9cb
commit 42e07dff55
3 changed files with 250 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
auto-bid-server/src/modules/mails/services/mails.service.ts
View File

@ -1,6 +1,7 @@
import { Bid } from '@/modules/bids/entities/bid.entity'; import { Bid } from '@/modules/bids/entities/bid.entity';
import { BidMetadatasService } from '@/modules/bids/services/bid-metadatas.service'; import { BidMetadatasService } from '@/modules/bids/services/bid-metadatas.service';
import { BidsService } from '@/modules/bids/services/bids.service'; import { BidsService } from '@/modules/bids/services/bids.service';
import { ScrapConfig } from '@/modules/scraps/entities/scrap-config.entity';
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
import { import {
extractDomain, extractDomain,
@ -59,6 +60,7 @@ export class MailsService {
} }
generateProductTableHTML( generateProductTableHTML(
keywords: { source: string; keywords: string }[],
products: ScrapItem[], products: ScrapItem[],
hotItems: { name: string }[], hotItems: { name: string }[],
): { ): {
@ -69,6 +71,24 @@ export class MailsService {
let hasHotItem = false; let hasHotItem = false;
const keywordHtml = keywords
.map((k) => {
return `
<span style="
display: inline-block;
color: #e03131;
padding: 4px 6px;
margin: 4px 6px 4px 0;
font-size: 12px;
font-weight: 600;
white-space: nowrap;
">
${k.source}: ${k.keywords}
</span>
`;
})
.join('');
if (!products.length) { if (!products.length) {
return { return {
html: ` html: `
@ -80,6 +100,27 @@ export class MailsService {
<title>Product Listing</title> <title>Product Listing</title>
</head> </head>
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;"> <body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
<div style="
text-align: center;
margin-bottom: 20px;
">
<div style="
margin-bottom: 8px;
font-size: 13px;
color: #555;
font-weight: 600;
">
🔎 Keywords matched today
</div>
<div style="
display: flex;
flex-wrap: wrap;
justify-content: center;
">
${keywordHtml}
</div>
</div>
<h2 style="text-align: center; color: #333;">Product Listing</h2> <h2 style="text-align: center; color: #333;">Product Listing</h2>
<p style="text-align: center; color: #666;">No matching products found for your keywords today.</p> <p style="text-align: center; color: #666;">No matching products found for your keywords today.</p>
<p style="text-align: center; color: #999; font-size: 12px; margin-top: 40px;">From: ${from}</p> <p style="text-align: center; color: #999; font-size: 12px; margin-top: 40px;">From: ${from}</p>
@ -160,6 +201,27 @@ export class MailsService {
<title>${title}</title> <title>${title}</title>
</head> </head>
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;"> <body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
<div style="
text-align: center;
margin-bottom: 20px;
">
<div style="
margin-bottom: 8px;
font-size: 13px;
color: #555;
font-weight: 600;
">
🔎 Keywords matched today
</div>
<div style="
display: flex;
flex-wrap: wrap;
justify-content: center;
">
${keywordHtml}
</div>
</div>
<h2 style="text-align: center; color: #333;">${title}</h2> <h2 style="text-align: center; color: #333;">${title}</h2>
<div style="overflow-x: auto;"> <div style="overflow-x: auto;">
<table style="width: 100%; min-width: 500px; border-collapse: collapse; background: #fff; border-radius: 6px; box-shadow: 0 2px 6px rgba(0,0,0,0.05);"> <table style="width: 100%; min-width: 500px; border-collapse: collapse; background: #fff; border-radius: 6px; box-shadow: 0 2px 6px rgba(0,0,0,0.05);">

153
auto-bid-server/src/modules/scraps/services/tasks.service.ts
View File

@ -1,13 +1,16 @@
import { ConfigsService } from '@/modules/bids/services/configs.service'; import { ConfigsService } from '@/modules/bids/services/configs.service';
import { DashboardService } from '@/modules/bids/services/dashboard.service'; import { DashboardService } from '@/modules/bids/services/dashboard.service';
import { MailsService } from '@/modules/mails/services/mails.service'; import { MailsService } from '@/modules/mails/services/mails.service';
import { delay } from '@/ultils'; import { delay, extractDomainSmart } from '@/ultils';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule'; import { Cron, CronExpression } from '@nestjs/schedule';
import * as moment from 'moment'; import * as moment from 'moment';
import { Between } from 'typeorm'; import { Between, In, Like, Not } from 'typeorm';
import { ScrapItemsService } from './scrap-item-config.service'; import { ScrapItemsService } from './scrap-item-config.service';
import { HotItemApi } from '@/modules/bids/apis/hot-item.api'; import { HotItemApi } from '@/modules/bids/apis/hot-item.api';
import { ScrapConfigsService } from './scrap-config.service';
import { capitalize } from 'lodash';
import { ScrapItem } from '../entities/scrap-item.entity';
@Injectable() @Injectable()
export class TasksService { export class TasksService {
@ -15,20 +18,14 @@ export class TasksService {
constructor( constructor(
private readonly scrapItemsService: ScrapItemsService, private readonly scrapItemsService: ScrapItemsService,
private readonly scrapConfigService: ScrapConfigsService,
private readonly mailsService: MailsService, private readonly mailsService: MailsService,
private readonly configsSerivce: ConfigsService, private readonly configsSerivce: ConfigsService,
private readonly dashboardService: DashboardService, private readonly dashboardService: DashboardService,
private readonly hotItemApi: HotItemApi, private readonly hotItemApi: HotItemApi,
) {} ) {}
async runProcessAndSendReport(processName: string) { async runProcess(processName: string) {
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
?.value;
if (!mails) {
console.warn('No mails configured for report. Skipping.');
return;
}
// Nếu process đang chạy, không chạy lại // Nếu process đang chạy, không chạy lại
const initialStatus = const initialStatus =
await this.dashboardService.getStatusProcessByName(processName); await this.dashboardService.getStatusProcessByName(processName);
@ -60,6 +57,15 @@ export class TasksService {
); );
return; return;
} }
}
async sendReportAllMail() {
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
?.value;
if (!mails) {
console.warn('No mails configured for report. Skipping.');
return;
}
// Khi process kết thúc => gửi mail // Khi process kết thúc => gửi mail
const startOfDay = new Date(); const startOfDay = new Date();
@ -76,9 +82,23 @@ export class TasksService {
order: { updated_at: 'DESC' }, order: { updated_at: 'DESC' },
}); });
const scrapConfigs = await this.scrapConfigService.scrapConfigRepo.find({
relations: { web_bid: true },
});
const keywords = scrapConfigs.map((scrapConfig) => {
return {
source: capitalize(
extractDomainSmart(scrapConfig.web_bid.origin_url),
),
keywords: scrapConfig.keywords,
};
});
const hotItems = await this.hotItemApi.listHotItem(); const hotItems = await this.hotItemApi.listHotItem();
const { hasHotItem, html } = this.mailsService.generateProductTableHTML( const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
keywords,
data, data,
hotItems, hotItems,
); );
@ -95,10 +115,119 @@ export class TasksService {
} }
} }
async sendReportWithConfig({
subject,
data,
keywords,
}: {
subject: string;
keywords: { source: string; keywords: string }[];
data: ScrapItem[];
}) {
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
?.value;
if (!mails) {
console.warn('No mails configured for report. Skipping.');
return;
}
try {
const hotItems = await this.hotItemApi.listHotItem();
const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
keywords,
data,
hotItems,
);
await this.mailsService.sendHtmlMailJob({
to: mails,
subject: `${hasHotItem ? '[HOT ITEMS] ' : ''} ${subject} – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`,
html: html,
});
console.log('Report mail sent successfully.');
} catch (err) {
console.error('Failed to generate or send report:', err);
}
}
// Thực hiện gọi scrape vào lúc 8h59 (giờ ÚC) mỗi ngày // Thực hiện gọi scrape vào lúc 8h59 (giờ ÚC) mỗi ngày
@Cron('59 5 * * *') @Cron(CronExpression.EVERY_MINUTE)
// @Cron('59 5 * * *')
async handleScraps() { async handleScraps() {
const processName = 'scrape-data-keyword'; const processName = 'scrape-data-keyword';
await this.runProcessAndSendReport(processName); // await this.runProcessAndSendReport(processName);
// await this.runProcess(processName);
const ramKeywords = ['DDR4', 'DDR5', 'DDR4|', 'DDR5|'];
// Khi process kết thúc => gửi mail
const startOfDay = new Date();
startOfDay.setHours(0, 0, 0, 0);
const endOfDay = new Date();
endOfDay.setHours(23, 59, 59, 999);
// Data Not Ram
const products = await this.scrapItemsService.scrapItemRepo.find({
where: {
updated_at: Between(startOfDay, endOfDay),
keyword: Not(In(ramKeywords)),
},
relations: { scrap_config: { web_bid: true } },
order: { updated_at: 'DESC' },
});
const qb = this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
ramKeywords.forEach((k, index) => {
qb.andWhere(`sc.keywords NOT LIKE :kw${index}`, {
[`kw${index}`]: `%${k}%`,
});
});
// Product Config
const productConfigs = await qb.getMany();
// Data Not Ram
const rams = await this.scrapItemsService.scrapItemRepo.find({
where: {
updated_at: Between(startOfDay, endOfDay),
keyword: In(ramKeywords),
},
relations: { scrap_config: { web_bid: true } },
order: { updated_at: 'DESC' },
});
// Ram Config
const ramConfigQb =
this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
ramKeywords.forEach((kw, i) => {
ramConfigQb.orWhere('sc.keywords LIKE :kw' + i, {
['kw' + i]: `%${kw}%`,
});
});
const ramConfigs = await ramConfigQb.getMany();
await this.sendReportWithConfig({
data: products,
keywords: productConfigs.map((item) => ({
source: extractDomainSmart(item.search_url),
keywords: item.keywords,
})),
subject: 'Auction Items Matching Your Keywords',
});
await this.sendReportWithConfig({
data: rams,
keywords: ramConfigs.map((item) => ({
source: extractDomainSmart(item.search_url),
keywords: item.keywords,
})),
subject: 'DDR4, DDR5 Item Report',
});
} }
} }

89
scrape-data-keyword/models/pickles-scrap-model.js
View File

@ -33,7 +33,7 @@ export class PicklesScrapModel extends ScrapModel {
const priceText = await newPage const priceText = await newPage
.$eval("#bidPrefix > span.font-weight-bold", (el) => .$eval("#bidPrefix > span.font-weight-bold", (el) =>
el.textContent.trim() el.textContent.trim(),
) )
.catch(() => null); .catch(() => null);
@ -50,7 +50,7 @@ export class PicklesScrapModel extends ScrapModel {
try { try {
const priceText = await elementHandle const priceText = await elementHandle
.$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) => .$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) =>
el.textContent.trim() el.textContent.trim(),
) )
.catch(() => null); .catch(() => null);
@ -61,48 +61,53 @@ export class PicklesScrapModel extends ScrapModel {
}; };
getItemsInHtml = async (data) => { getItemsInHtml = async (data) => {
await this.page.waitForSelector( try {
".content-wrapper_contentgridwrapper__3RCQZ > div.column", await this.page.waitForSelector(
{ ".content-wrapper_contentgridwrapper__3RCQZ > div.column",
timeout: 10000, {
timeout: 10000,
},
);
const elements = await this.page.$$(
".content-wrapper_contentgridwrapper__3RCQZ > div.column",
);
const results = [];
for (const el of elements) {
const url = await el
.$eval("main > a", (el) => el.getAttribute("href"))
.catch(() => null);
const image_url = await el
.$eval("div > div:first-child > div > div:first-child > img", (img) =>
img.getAttribute("src"),
)
.catch(() => null);
const name = await el
.$eval("header > h2:first-of-type", (el) => el.textContent.trim())
.catch(() => null);
const model = extractModelId(`${this.web_bid.origin_url}${url}`);
const current_price = await this.getPriceByEl(el, model);
results.push({
url: `${this.web_bid.origin_url}${url}`,
image_url,
name,
keyword: data.keyword,
model,
current_price: current_price,
scrap_config_id: this.scrap_config_id,
});
} }
);
const elements = await this.page.$$( return results;
".content-wrapper_contentgridwrapper__3RCQZ > div.column" } catch (error) {
); return [];
const results = [];
for (const el of elements) {
const url = await el
.$eval("main > a", (el) => el.getAttribute("href"))
.catch(() => null);
const image_url = await el
.$eval("div > div:first-child > div > div:first-child > img", (img) =>
img.getAttribute("src")
)
.catch(() => null);
const name = await el
.$eval("header > h2:first-of-type", (el) => el.textContent.trim())
.catch(() => null);
const model = extractModelId(`${this.web_bid.origin_url}${url}`);
const current_price = await this.getPriceByEl(el, model);
results.push({
url: `${this.web_bid.origin_url}${url}`,
image_url,
name,
keyword: data.keyword,
model,
current_price: current_price,
scrap_config_id: this.scrap_config_id,
});
} }
return results;
}; };
} }