zelda
/
bid-tool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extract content ram product to a new mail

This commit is contained in:
Admin 2026-01-30 11:03:52 +07:00
parent f4b677e9cb
commit 42e07dff55
3 changed files with 250 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
auto-bid-server/src/modules/mails/services/mails.service.ts
View File

@ -1,6 +1,7 @@
import { Bid } from '@/modules/bids/entities/bid.entity';
import { BidMetadatasService } from '@/modules/bids/services/bid-metadatas.service';
import { BidsService } from '@/modules/bids/services/bids.service';
import { ScrapConfig } from '@/modules/scraps/entities/scrap-config.entity';
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
import {
extractDomain,
@ -59,6 +60,7 @@ export class MailsService {
}
generateProductTableHTML(
keywords: { source: string; keywords: string }[],
products: ScrapItem[],
hotItems: { name: string }[],
): {
@ -69,6 +71,24 @@ export class MailsService {
let hasHotItem = false;
const keywordHtml = keywords
.map((k) => {
return `
<span style="
display: inline-block;
color: #e03131;
padding: 4px 6px;
margin: 4px 6px 4px 0;
font-size: 12px;
font-weight: 600;
white-space: nowrap;
">
${k.source}: ${k.keywords}
</span>
`;
})
.join('');
if (!products.length) {
return {
html: `
@ -80,6 +100,27 @@ export class MailsService {
<title>Product Listing</title>
</head>
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
<div style="
text-align: center;
margin-bottom: 20px;
">
<div style="
margin-bottom: 8px;
font-size: 13px;
color: #555;
font-weight: 600;
">
🔎 Keywords matched today
</div>
<div style="
display: flex;
flex-wrap: wrap;
justify-content: center;
">
${keywordHtml}
</div>
</div>
<h2 style="text-align: center; color: #333;">Product Listing</h2>
<p style="text-align: center; color: #666;">No matching products found for your keywords today.</p>
<p style="text-align: center; color: #999; font-size: 12px; margin-top: 40px;">From: ${from}</p>
@ -160,6 +201,27 @@ export class MailsService {
<title>${title}</title>
</head>
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
<div style="
text-align: center;
margin-bottom: 20px;
">
<div style="
margin-bottom: 8px;
font-size: 13px;
color: #555;
font-weight: 600;
">
🔎 Keywords matched today
</div>
<div style="
display: flex;
flex-wrap: wrap;
justify-content: center;
">
${keywordHtml}
</div>
</div>
<h2 style="text-align: center; color: #333;">${title}</h2>
<div style="overflow-x: auto;">
<table style="width: 100%; min-width: 500px; border-collapse: collapse; background: #fff; border-radius: 6px; box-shadow: 0 2px 6px rgba(0,0,0,0.05);">

153
auto-bid-server/src/modules/scraps/services/tasks.service.ts
View File

@ -1,13 +1,16 @@
import { ConfigsService } from '@/modules/bids/services/configs.service';
import { DashboardService } from '@/modules/bids/services/dashboard.service';
import { MailsService } from '@/modules/mails/services/mails.service';
import { delay } from '@/ultils';
import { delay, extractDomainSmart } from '@/ultils';
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import * as moment from 'moment';
import { Between } from 'typeorm';
import { Between, In, Like, Not } from 'typeorm';
import { ScrapItemsService } from './scrap-item-config.service';
import { HotItemApi } from '@/modules/bids/apis/hot-item.api';
import { ScrapConfigsService } from './scrap-config.service';
import { capitalize } from 'lodash';
import { ScrapItem } from '../entities/scrap-item.entity';
@Injectable()
export class TasksService {
@ -15,20 +18,14 @@ export class TasksService {
constructor(
private readonly scrapItemsService: ScrapItemsService,
private readonly scrapConfigService: ScrapConfigsService,
private readonly mailsService: MailsService,
private readonly configsSerivce: ConfigsService,
private readonly dashboardService: DashboardService,
private readonly hotItemApi: HotItemApi,
) {}
async runProcessAndSendReport(processName: string) {
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
?.value;
if (!mails) {
console.warn('No mails configured for report. Skipping.');
return;
}
async runProcess(processName: string) {
// Nếu process đang chạy, không chạy lại
const initialStatus =
await this.dashboardService.getStatusProcessByName(processName);
@ -60,6 +57,15 @@ export class TasksService {
);
return;
}
}
async sendReportAllMail() {
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
?.value;
if (!mails) {
console.warn('No mails configured for report. Skipping.');
return;
}
// Khi process kết thúc => gửi mail
const startOfDay = new Date();
@ -76,9 +82,23 @@ export class TasksService {
order: { updated_at: 'DESC' },
});
const scrapConfigs = await this.scrapConfigService.scrapConfigRepo.find({
relations: { web_bid: true },
});
const keywords = scrapConfigs.map((scrapConfig) => {
return {
source: capitalize(
extractDomainSmart(scrapConfig.web_bid.origin_url),
),
keywords: scrapConfig.keywords,
};
});
const hotItems = await this.hotItemApi.listHotItem();
const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
keywords,
data,
hotItems,
);
@ -95,10 +115,119 @@ export class TasksService {
}
}
async sendReportWithConfig({
subject,
data,
keywords,
}: {
subject: string;
keywords: { source: string; keywords: string }[];
data: ScrapItem[];
}) {
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
?.value;
if (!mails) {
console.warn('No mails configured for report. Skipping.');
return;
}
try {
const hotItems = await this.hotItemApi.listHotItem();
const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
keywords,
data,
hotItems,
);
await this.mailsService.sendHtmlMailJob({
to: mails,
subject: `${hasHotItem ? '[HOT ITEMS] ' : ''} ${subject} – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`,
html: html,
});
console.log('Report mail sent successfully.');
} catch (err) {
console.error('Failed to generate or send report:', err);
}
}
// Thực hiện gọi scrape vào lúc 8h59 (giờ ÚC) mỗi ngày
@Cron('59 5 * * *')
@Cron(CronExpression.EVERY_MINUTE)
// @Cron('59 5 * * *')
async handleScraps() {
const processName = 'scrape-data-keyword';
await this.runProcessAndSendReport(processName);
// await this.runProcessAndSendReport(processName);
// await this.runProcess(processName);
const ramKeywords = ['DDR4', 'DDR5', 'DDR4|', 'DDR5|'];
// Khi process kết thúc => gửi mail
const startOfDay = new Date();
startOfDay.setHours(0, 0, 0, 0);
const endOfDay = new Date();
endOfDay.setHours(23, 59, 59, 999);
// Data Not Ram
const products = await this.scrapItemsService.scrapItemRepo.find({
where: {
updated_at: Between(startOfDay, endOfDay),
keyword: Not(In(ramKeywords)),
},
relations: { scrap_config: { web_bid: true } },
order: { updated_at: 'DESC' },
});
const qb = this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
ramKeywords.forEach((k, index) => {
qb.andWhere(`sc.keywords NOT LIKE :kw${index}`, {
[`kw${index}`]: `%${k}%`,
});
});
// Product Config
const productConfigs = await qb.getMany();
// Data Not Ram
const rams = await this.scrapItemsService.scrapItemRepo.find({
where: {
updated_at: Between(startOfDay, endOfDay),
keyword: In(ramKeywords),
},
relations: { scrap_config: { web_bid: true } },
order: { updated_at: 'DESC' },
});
// Ram Config
const ramConfigQb =
this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
ramKeywords.forEach((kw, i) => {
ramConfigQb.orWhere('sc.keywords LIKE :kw' + i, {
['kw' + i]: `%${kw}%`,
});
});
const ramConfigs = await ramConfigQb.getMany();
await this.sendReportWithConfig({
data: products,
keywords: productConfigs.map((item) => ({
source: extractDomainSmart(item.search_url),
keywords: item.keywords,
})),
subject: 'Auction Items Matching Your Keywords',
});
await this.sendReportWithConfig({
data: rams,
keywords: ramConfigs.map((item) => ({
source: extractDomainSmart(item.search_url),
keywords: item.keywords,
})),
subject: 'DDR4, DDR5 Item Report',
});
}
}

15
scrape-data-keyword/models/pickles-scrap-model.js
View File

@ -33,7 +33,7 @@ export class PicklesScrapModel extends ScrapModel {
const priceText = await newPage
.$eval("#bidPrefix > span.font-weight-bold", (el) =>
el.textContent.trim()
el.textContent.trim(),
)
.catch(() => null);
@ -50,7 +50,7 @@ export class PicklesScrapModel extends ScrapModel {
try {
const priceText = await elementHandle
.$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) =>
el.textContent.trim()
el.textContent.trim(),
)
.catch(() => null);
@ -61,16 +61,18 @@ export class PicklesScrapModel extends ScrapModel {
};
getItemsInHtml = async (data) => {
try {
await this.page.waitForSelector(
".content-wrapper_contentgridwrapper__3RCQZ > div.column",
{
timeout: 10000,
}
},
);
const elements = await this.page.$$(
".content-wrapper_contentgridwrapper__3RCQZ > div.column"
".content-wrapper_contentgridwrapper__3RCQZ > div.column",
);
const results = [];
for (const el of elements) {
@ -80,7 +82,7 @@ export class PicklesScrapModel extends ScrapModel {
const image_url = await el
.$eval("div > div:first-child > div > div:first-child > img", (img) =>
img.getAttribute("src")
img.getAttribute("src"),
)
.catch(() => null);
@ -104,5 +106,8 @@ export class PicklesScrapModel extends ScrapModel {
}
return results;
} catch (error) {
return [];
}
};
}