extract content ram product to a new mail
This commit is contained in:
Admin
parent
f4b677e9cb
commit
42e07dff55
|
|
@ -1,6 +1,7 @@
|
|||
import { Bid } from '@/modules/bids/entities/bid.entity';
|
||||
import { BidMetadatasService } from '@/modules/bids/services/bid-metadatas.service';
|
||||
import { BidsService } from '@/modules/bids/services/bids.service';
|
||||
import { ScrapConfig } from '@/modules/scraps/entities/scrap-config.entity';
|
||||
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
|
||||
import {
|
||||
extractDomain,
|
||||
|
|
@ -59,6 +60,7 @@ export class MailsService {
|
|||
}
|
||||
|
||||
generateProductTableHTML(
|
||||
keywords: { source: string; keywords: string }[],
|
||||
products: ScrapItem[],
|
||||
hotItems: { name: string }[],
|
||||
): {
|
||||
|
|
@ -69,6 +71,24 @@ export class MailsService {
|
|||
|
||||
let hasHotItem = false;
|
||||
|
||||
const keywordHtml = keywords
|
||||
.map((k) => {
|
||||
return `
|
||||
<span style="
|
||||
display: inline-block;
|
||||
color: #e03131;
|
||||
padding: 4px 6px;
|
||||
margin: 4px 6px 4px 0;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
">
|
||||
${k.source}: ${k.keywords}
|
||||
</span>
|
||||
`;
|
||||
})
|
||||
.join('');
|
||||
|
||||
if (!products.length) {
|
||||
return {
|
||||
html: `
|
||||
|
|
@ -80,6 +100,27 @@ export class MailsService {
|
|||
<title>Product Listing</title>
|
||||
</head>
|
||||
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
|
||||
<div style="
|
||||
text-align: center;
|
||||
margin-bottom: 20px;
|
||||
">
|
||||
<div style="
|
||||
margin-bottom: 8px;
|
||||
font-size: 13px;
|
||||
color: #555;
|
||||
font-weight: 600;
|
||||
">
|
||||
🔎 Keywords matched today
|
||||
</div>
|
||||
|
||||
<div style="
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
">
|
||||
${keywordHtml}
|
||||
</div>
|
||||
</div>
|
||||
<h2 style="text-align: center; color: #333;">Product Listing</h2>
|
||||
<p style="text-align: center; color: #666;">No matching products found for your keywords today.</p>
|
||||
<p style="text-align: center; color: #999; font-size: 12px; margin-top: 40px;">From: ${from}</p>
|
||||
|
|
@ -160,6 +201,27 @@ export class MailsService {
|
|||
<title>${title}</title>
|
||||
</head>
|
||||
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
|
||||
<div style="
|
||||
text-align: center;
|
||||
margin-bottom: 20px;
|
||||
">
|
||||
<div style="
|
||||
margin-bottom: 8px;
|
||||
font-size: 13px;
|
||||
color: #555;
|
||||
font-weight: 600;
|
||||
">
|
||||
🔎 Keywords matched today
|
||||
</div>
|
||||
|
||||
<div style="
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
">
|
||||
${keywordHtml}
|
||||
</div>
|
||||
</div>
|
||||
<h2 style="text-align: center; color: #333;">${title}</h2>
|
||||
<div style="overflow-x: auto;">
|
||||
<table style="width: 100%; min-width: 500px; border-collapse: collapse; background: #fff; border-radius: 6px; box-shadow: 0 2px 6px rgba(0,0,0,0.05);">
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
import { ConfigsService } from '@/modules/bids/services/configs.service';
|
||||
import { DashboardService } from '@/modules/bids/services/dashboard.service';
|
||||
import { MailsService } from '@/modules/mails/services/mails.service';
|
||||
import { delay } from '@/ultils';
|
||||
import { delay, extractDomainSmart } from '@/ultils';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import * as moment from 'moment';
|
||||
import { Between } from 'typeorm';
|
||||
import { Between, In, Like, Not } from 'typeorm';
|
||||
import { ScrapItemsService } from './scrap-item-config.service';
|
||||
import { HotItemApi } from '@/modules/bids/apis/hot-item.api';
|
||||
import { ScrapConfigsService } from './scrap-config.service';
|
||||
import { capitalize } from 'lodash';
|
||||
import { ScrapItem } from '../entities/scrap-item.entity';
|
||||
|
||||
@Injectable()
|
||||
export class TasksService {
|
||||
|
|
@ -15,20 +18,14 @@ export class TasksService {
|
|||
|
||||
constructor(
|
||||
private readonly scrapItemsService: ScrapItemsService,
|
||||
private readonly scrapConfigService: ScrapConfigsService,
|
||||
private readonly mailsService: MailsService,
|
||||
private readonly configsSerivce: ConfigsService,
|
||||
private readonly dashboardService: DashboardService,
|
||||
private readonly hotItemApi: HotItemApi,
|
||||
) {}
|
||||
|
||||
async runProcessAndSendReport(processName: string) {
|
||||
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
|
||||
?.value;
|
||||
if (!mails) {
|
||||
console.warn('No mails configured for report. Skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
async runProcess(processName: string) {
|
||||
// Nếu process đang chạy, không chạy lại
|
||||
const initialStatus =
|
||||
await this.dashboardService.getStatusProcessByName(processName);
|
||||
|
|
@ -60,6 +57,15 @@ export class TasksService {
|
|||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async sendReportAllMail() {
|
||||
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
|
||||
?.value;
|
||||
if (!mails) {
|
||||
console.warn('No mails configured for report. Skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Khi process kết thúc => gửi mail
|
||||
const startOfDay = new Date();
|
||||
|
|
@ -76,9 +82,23 @@ export class TasksService {
|
|||
order: { updated_at: 'DESC' },
|
||||
});
|
||||
|
||||
const scrapConfigs = await this.scrapConfigService.scrapConfigRepo.find({
|
||||
relations: { web_bid: true },
|
||||
});
|
||||
|
||||
const keywords = scrapConfigs.map((scrapConfig) => {
|
||||
return {
|
||||
source: capitalize(
|
||||
extractDomainSmart(scrapConfig.web_bid.origin_url),
|
||||
),
|
||||
keywords: scrapConfig.keywords,
|
||||
};
|
||||
});
|
||||
|
||||
const hotItems = await this.hotItemApi.listHotItem();
|
||||
|
||||
const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
|
||||
keywords,
|
||||
data,
|
||||
hotItems,
|
||||
);
|
||||
|
|
@ -95,10 +115,119 @@ export class TasksService {
|
|||
}
|
||||
}
|
||||
|
||||
async sendReportWithConfig({
|
||||
subject,
|
||||
data,
|
||||
keywords,
|
||||
}: {
|
||||
subject: string;
|
||||
keywords: { source: string; keywords: string }[];
|
||||
data: ScrapItem[];
|
||||
}) {
|
||||
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
|
||||
?.value;
|
||||
if (!mails) {
|
||||
console.warn('No mails configured for report. Skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const hotItems = await this.hotItemApi.listHotItem();
|
||||
|
||||
const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
|
||||
keywords,
|
||||
data,
|
||||
hotItems,
|
||||
);
|
||||
|
||||
await this.mailsService.sendHtmlMailJob({
|
||||
to: mails,
|
||||
subject: `${hasHotItem ? '[HOT ITEMS] ' : ''} ${subject} – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`,
|
||||
html: html,
|
||||
});
|
||||
|
||||
console.log('Report mail sent successfully.');
|
||||
} catch (err) {
|
||||
console.error('Failed to generate or send report:', err);
|
||||
}
|
||||
}
|
||||
|
||||
// Thực hiện gọi scrape vào lúc 8h59 (giờ ÚC) mỗi ngày
|
||||
@Cron('59 5 * * *')
|
||||
@Cron(CronExpression.EVERY_MINUTE)
|
||||
// @Cron('59 5 * * *')
|
||||
async handleScraps() {
|
||||
const processName = 'scrape-data-keyword';
|
||||
await this.runProcessAndSendReport(processName);
|
||||
// await this.runProcessAndSendReport(processName);
|
||||
|
||||
// await this.runProcess(processName);
|
||||
|
||||
const ramKeywords = ['DDR4', 'DDR5', 'DDR4|', 'DDR5|'];
|
||||
|
||||
// Khi process kết thúc => gửi mail
|
||||
const startOfDay = new Date();
|
||||
startOfDay.setHours(0, 0, 0, 0);
|
||||
const endOfDay = new Date();
|
||||
endOfDay.setHours(23, 59, 59, 999);
|
||||
|
||||
// Data Not Ram
|
||||
const products = await this.scrapItemsService.scrapItemRepo.find({
|
||||
where: {
|
||||
updated_at: Between(startOfDay, endOfDay),
|
||||
keyword: Not(In(ramKeywords)),
|
||||
},
|
||||
relations: { scrap_config: { web_bid: true } },
|
||||
order: { updated_at: 'DESC' },
|
||||
});
|
||||
|
||||
const qb = this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
|
||||
|
||||
ramKeywords.forEach((k, index) => {
|
||||
qb.andWhere(`sc.keywords NOT LIKE :kw${index}`, {
|
||||
[`kw${index}`]: `%${k}%`,
|
||||
});
|
||||
});
|
||||
|
||||
// Product Config
|
||||
const productConfigs = await qb.getMany();
|
||||
|
||||
// Data Not Ram
|
||||
const rams = await this.scrapItemsService.scrapItemRepo.find({
|
||||
where: {
|
||||
updated_at: Between(startOfDay, endOfDay),
|
||||
keyword: In(ramKeywords),
|
||||
},
|
||||
relations: { scrap_config: { web_bid: true } },
|
||||
order: { updated_at: 'DESC' },
|
||||
});
|
||||
|
||||
// Ram Config
|
||||
const ramConfigQb =
|
||||
this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
|
||||
|
||||
ramKeywords.forEach((kw, i) => {
|
||||
ramConfigQb.orWhere('sc.keywords LIKE :kw' + i, {
|
||||
['kw' + i]: `%${kw}%`,
|
||||
});
|
||||
});
|
||||
|
||||
const ramConfigs = await ramConfigQb.getMany();
|
||||
|
||||
await this.sendReportWithConfig({
|
||||
data: products,
|
||||
keywords: productConfigs.map((item) => ({
|
||||
source: extractDomainSmart(item.search_url),
|
||||
keywords: item.keywords,
|
||||
})),
|
||||
subject: 'Auction Items Matching Your Keywords',
|
||||
});
|
||||
|
||||
await this.sendReportWithConfig({
|
||||
data: rams,
|
||||
keywords: ramConfigs.map((item) => ({
|
||||
source: extractDomainSmart(item.search_url),
|
||||
keywords: item.keywords,
|
||||
})),
|
||||
subject: 'DDR4, DDR5 Item Report',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ export class PicklesScrapModel extends ScrapModel {
|
|||
|
||||
const priceText = await newPage
|
||||
.$eval("#bidPrefix > span.font-weight-bold", (el) =>
|
||||
el.textContent.trim()
|
||||
el.textContent.trim(),
|
||||
)
|
||||
.catch(() => null);
|
||||
|
||||
|
|
@ -50,7 +50,7 @@ export class PicklesScrapModel extends ScrapModel {
|
|||
try {
|
||||
const priceText = await elementHandle
|
||||
.$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) =>
|
||||
el.textContent.trim()
|
||||
el.textContent.trim(),
|
||||
)
|
||||
.catch(() => null);
|
||||
|
||||
|
|
@ -61,48 +61,53 @@ export class PicklesScrapModel extends ScrapModel {
|
|||
};
|
||||
|
||||
getItemsInHtml = async (data) => {
|
||||
await this.page.waitForSelector(
|
||||
".content-wrapper_contentgridwrapper__3RCQZ > div.column",
|
||||
{
|
||||
timeout: 10000,
|
||||
try {
|
||||
await this.page.waitForSelector(
|
||||
".content-wrapper_contentgridwrapper__3RCQZ > div.column",
|
||||
{
|
||||
timeout: 10000,
|
||||
},
|
||||
);
|
||||
|
||||
const elements = await this.page.$$(
|
||||
".content-wrapper_contentgridwrapper__3RCQZ > div.column",
|
||||
);
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const el of elements) {
|
||||
const url = await el
|
||||
.$eval("main > a", (el) => el.getAttribute("href"))
|
||||
.catch(() => null);
|
||||
|
||||
const image_url = await el
|
||||
.$eval("div > div:first-child > div > div:first-child > img", (img) =>
|
||||
img.getAttribute("src"),
|
||||
)
|
||||
.catch(() => null);
|
||||
|
||||
const name = await el
|
||||
.$eval("header > h2:first-of-type", (el) => el.textContent.trim())
|
||||
.catch(() => null);
|
||||
|
||||
const model = extractModelId(`${this.web_bid.origin_url}${url}`);
|
||||
|
||||
const current_price = await this.getPriceByEl(el, model);
|
||||
|
||||
results.push({
|
||||
url: `${this.web_bid.origin_url}${url}`,
|
||||
image_url,
|
||||
name,
|
||||
keyword: data.keyword,
|
||||
model,
|
||||
current_price: current_price,
|
||||
scrap_config_id: this.scrap_config_id,
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
const elements = await this.page.$$(
|
||||
".content-wrapper_contentgridwrapper__3RCQZ > div.column"
|
||||
);
|
||||
const results = [];
|
||||
|
||||
for (const el of elements) {
|
||||
const url = await el
|
||||
.$eval("main > a", (el) => el.getAttribute("href"))
|
||||
.catch(() => null);
|
||||
|
||||
const image_url = await el
|
||||
.$eval("div > div:first-child > div > div:first-child > img", (img) =>
|
||||
img.getAttribute("src")
|
||||
)
|
||||
.catch(() => null);
|
||||
|
||||
const name = await el
|
||||
.$eval("header > h2:first-of-type", (el) => el.textContent.trim())
|
||||
.catch(() => null);
|
||||
|
||||
const model = extractModelId(`${this.web_bid.origin_url}${url}`);
|
||||
|
||||
const current_price = await this.getPriceByEl(el, model);
|
||||
|
||||
results.push({
|
||||
url: `${this.web_bid.origin_url}${url}`,
|
||||
image_url,
|
||||
name,
|
||||
keyword: data.keyword,
|
||||
model,
|
||||
current_price: current_price,
|
||||
scrap_config_id: this.scrap_config_id,
|
||||
});
|
||||
return results;
|
||||
} catch (error) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return results;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue