From 42e07dff55119d4ad9dd68057683297f73a9e84f Mon Sep 17 00:00:00 2001 From: Admin Date: Fri, 30 Jan 2026 11:03:52 +0700 Subject: [PATCH] extract content ram product to a new mail --- .../modules/mails/services/mails.service.ts | 62 +++++++ .../modules/scraps/services/tasks.service.ts | 153 ++++++++++++++++-- .../models/pickles-scrap-model.js | 89 +++++----- 3 files changed, 250 insertions(+), 54 deletions(-) diff --git a/auto-bid-server/src/modules/mails/services/mails.service.ts b/auto-bid-server/src/modules/mails/services/mails.service.ts index e711e7d..9fe68d4 100644 --- a/auto-bid-server/src/modules/mails/services/mails.service.ts +++ b/auto-bid-server/src/modules/mails/services/mails.service.ts @@ -1,6 +1,7 @@ import { Bid } from '@/modules/bids/entities/bid.entity'; import { BidMetadatasService } from '@/modules/bids/services/bid-metadatas.service'; import { BidsService } from '@/modules/bids/services/bids.service'; +import { ScrapConfig } from '@/modules/scraps/entities/scrap-config.entity'; import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; import { extractDomain, @@ -59,6 +60,7 @@ export class MailsService { } generateProductTableHTML( + keywords: { source: string; keywords: string }[], products: ScrapItem[], hotItems: { name: string }[], ): { @@ -69,6 +71,24 @@ export class MailsService { let hasHotItem = false; + const keywordHtml = keywords + .map((k) => { + return ` + + ${k.source}: ${k.keywords} + + `; + }) + .join(''); + if (!products.length) { return { html: ` @@ -80,6 +100,27 @@ export class MailsService { Product Listing +
+
+ 🔎 Keywords matched today +
+ +
+ ${keywordHtml} +
+

Product Listing

No matching products found for your keywords today.

From: ${from}

@@ -160,6 +201,27 @@ export class MailsService { ${title} +
+
+ 🔎 Keywords matched today +
+ +
+ ${keywordHtml} +
+

${title}

diff --git a/auto-bid-server/src/modules/scraps/services/tasks.service.ts b/auto-bid-server/src/modules/scraps/services/tasks.service.ts index 21a4fee..44ad490 100644 --- a/auto-bid-server/src/modules/scraps/services/tasks.service.ts +++ b/auto-bid-server/src/modules/scraps/services/tasks.service.ts @@ -1,13 +1,16 @@ import { ConfigsService } from '@/modules/bids/services/configs.service'; import { DashboardService } from '@/modules/bids/services/dashboard.service'; import { MailsService } from '@/modules/mails/services/mails.service'; -import { delay } from '@/ultils'; +import { delay, extractDomainSmart } from '@/ultils'; import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import * as moment from 'moment'; -import { Between } from 'typeorm'; +import { Between, In, Like, Not } from 'typeorm'; import { ScrapItemsService } from './scrap-item-config.service'; import { HotItemApi } from '@/modules/bids/apis/hot-item.api'; +import { ScrapConfigsService } from './scrap-config.service'; +import { capitalize } from 'lodash'; +import { ScrapItem } from '../entities/scrap-item.entity'; @Injectable() export class TasksService { @@ -15,20 +18,14 @@ export class TasksService { constructor( private readonly scrapItemsService: ScrapItemsService, + private readonly scrapConfigService: ScrapConfigsService, private readonly mailsService: MailsService, private readonly configsSerivce: ConfigsService, private readonly dashboardService: DashboardService, private readonly hotItemApi: HotItemApi, ) {} - async runProcessAndSendReport(processName: string) { - const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT')) - ?.value; - if (!mails) { - console.warn('No mails configured for report. Skipping.'); - return; - } - + async runProcess(processName: string) { // Nếu process đang chạy, không chạy lại const initialStatus = await this.dashboardService.getStatusProcessByName(processName); @@ -60,6 +57,15 @@ export class TasksService { ); return; } + } + + async sendReportAllMail() { + const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT')) + ?.value; + if (!mails) { + console.warn('No mails configured for report. Skipping.'); + return; + } // Khi process kết thúc => gửi mail const startOfDay = new Date(); @@ -76,9 +82,23 @@ export class TasksService { order: { updated_at: 'DESC' }, }); + const scrapConfigs = await this.scrapConfigService.scrapConfigRepo.find({ + relations: { web_bid: true }, + }); + + const keywords = scrapConfigs.map((scrapConfig) => { + return { + source: capitalize( + extractDomainSmart(scrapConfig.web_bid.origin_url), + ), + keywords: scrapConfig.keywords, + }; + }); + const hotItems = await this.hotItemApi.listHotItem(); const { hasHotItem, html } = this.mailsService.generateProductTableHTML( + keywords, data, hotItems, ); @@ -95,10 +115,119 @@ export class TasksService { } } + async sendReportWithConfig({ + subject, + data, + keywords, + }: { + subject: string; + keywords: { source: string; keywords: string }[]; + data: ScrapItem[]; + }) { + const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT')) + ?.value; + if (!mails) { + console.warn('No mails configured for report. Skipping.'); + return; + } + + try { + const hotItems = await this.hotItemApi.listHotItem(); + + const { hasHotItem, html } = this.mailsService.generateProductTableHTML( + keywords, + data, + hotItems, + ); + + await this.mailsService.sendHtmlMailJob({ + to: mails, + subject: `${hasHotItem ? '[HOT ITEMS] ' : ''} ${subject} – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`, + html: html, + }); + + console.log('Report mail sent successfully.'); + } catch (err) { + console.error('Failed to generate or send report:', err); + } + } + // Thực hiện gọi scrape vào lúc 8h59 (giờ ÚC) mỗi ngày - @Cron('59 5 * * *') + @Cron(CronExpression.EVERY_MINUTE) + // @Cron('59 5 * * *') async handleScraps() { const processName = 'scrape-data-keyword'; - await this.runProcessAndSendReport(processName); + // await this.runProcessAndSendReport(processName); + + // await this.runProcess(processName); + + const ramKeywords = ['DDR4', 'DDR5', 'DDR4|', 'DDR5|']; + + // Khi process kết thúc => gửi mail + const startOfDay = new Date(); + startOfDay.setHours(0, 0, 0, 0); + const endOfDay = new Date(); + endOfDay.setHours(23, 59, 59, 999); + + // Data Not Ram + const products = await this.scrapItemsService.scrapItemRepo.find({ + where: { + updated_at: Between(startOfDay, endOfDay), + keyword: Not(In(ramKeywords)), + }, + relations: { scrap_config: { web_bid: true } }, + order: { updated_at: 'DESC' }, + }); + + const qb = this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc'); + + ramKeywords.forEach((k, index) => { + qb.andWhere(`sc.keywords NOT LIKE :kw${index}`, { + [`kw${index}`]: `%${k}%`, + }); + }); + + // Product Config + const productConfigs = await qb.getMany(); + + // Data Not Ram + const rams = await this.scrapItemsService.scrapItemRepo.find({ + where: { + updated_at: Between(startOfDay, endOfDay), + keyword: In(ramKeywords), + }, + relations: { scrap_config: { web_bid: true } }, + order: { updated_at: 'DESC' }, + }); + + // Ram Config + const ramConfigQb = + this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc'); + + ramKeywords.forEach((kw, i) => { + ramConfigQb.orWhere('sc.keywords LIKE :kw' + i, { + ['kw' + i]: `%${kw}%`, + }); + }); + + const ramConfigs = await ramConfigQb.getMany(); + + await this.sendReportWithConfig({ + data: products, + keywords: productConfigs.map((item) => ({ + source: extractDomainSmart(item.search_url), + keywords: item.keywords, + })), + subject: 'Auction Items Matching Your Keywords', + }); + + await this.sendReportWithConfig({ + data: rams, + keywords: ramConfigs.map((item) => ({ + source: extractDomainSmart(item.search_url), + keywords: item.keywords, + })), + subject: 'DDR4, DDR5 Item Report', + }); } } diff --git a/scrape-data-keyword/models/pickles-scrap-model.js b/scrape-data-keyword/models/pickles-scrap-model.js index 7874cf4..ce4d2e6 100644 --- a/scrape-data-keyword/models/pickles-scrap-model.js +++ b/scrape-data-keyword/models/pickles-scrap-model.js @@ -33,7 +33,7 @@ export class PicklesScrapModel extends ScrapModel { const priceText = await newPage .$eval("#bidPrefix > span.font-weight-bold", (el) => - el.textContent.trim() + el.textContent.trim(), ) .catch(() => null); @@ -50,7 +50,7 @@ export class PicklesScrapModel extends ScrapModel { try { const priceText = await elementHandle .$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) => - el.textContent.trim() + el.textContent.trim(), ) .catch(() => null); @@ -61,48 +61,53 @@ export class PicklesScrapModel extends ScrapModel { }; getItemsInHtml = async (data) => { - await this.page.waitForSelector( - ".content-wrapper_contentgridwrapper__3RCQZ > div.column", - { - timeout: 10000, + try { + await this.page.waitForSelector( + ".content-wrapper_contentgridwrapper__3RCQZ > div.column", + { + timeout: 10000, + }, + ); + + const elements = await this.page.$$( + ".content-wrapper_contentgridwrapper__3RCQZ > div.column", + ); + + const results = []; + + for (const el of elements) { + const url = await el + .$eval("main > a", (el) => el.getAttribute("href")) + .catch(() => null); + + const image_url = await el + .$eval("div > div:first-child > div > div:first-child > img", (img) => + img.getAttribute("src"), + ) + .catch(() => null); + + const name = await el + .$eval("header > h2:first-of-type", (el) => el.textContent.trim()) + .catch(() => null); + + const model = extractModelId(`${this.web_bid.origin_url}${url}`); + + const current_price = await this.getPriceByEl(el, model); + + results.push({ + url: `${this.web_bid.origin_url}${url}`, + image_url, + name, + keyword: data.keyword, + model, + current_price: current_price, + scrap_config_id: this.scrap_config_id, + }); } - ); - const elements = await this.page.$$( - ".content-wrapper_contentgridwrapper__3RCQZ > div.column" - ); - const results = []; - - for (const el of elements) { - const url = await el - .$eval("main > a", (el) => el.getAttribute("href")) - .catch(() => null); - - const image_url = await el - .$eval("div > div:first-child > div > div:first-child > img", (img) => - img.getAttribute("src") - ) - .catch(() => null); - - const name = await el - .$eval("header > h2:first-of-type", (el) => el.textContent.trim()) - .catch(() => null); - - const model = extractModelId(`${this.web_bid.origin_url}${url}`); - - const current_price = await this.getPriceByEl(el, model); - - results.push({ - url: `${this.web_bid.origin_url}${url}`, - image_url, - name, - keyword: data.keyword, - model, - current_price: current_price, - scrap_config_id: this.scrap_config_id, - }); + return results; + } catch (error) { + return []; } - - return results; }; }