diff --git a/auto-bid-server/src/modules/mails/services/mails.service.ts b/auto-bid-server/src/modules/mails/services/mails.service.ts
index e711e7d..9fe68d4 100644
--- a/auto-bid-server/src/modules/mails/services/mails.service.ts
+++ b/auto-bid-server/src/modules/mails/services/mails.service.ts
@@ -1,6 +1,7 @@
import { Bid } from '@/modules/bids/entities/bid.entity';
import { BidMetadatasService } from '@/modules/bids/services/bid-metadatas.service';
import { BidsService } from '@/modules/bids/services/bids.service';
+import { ScrapConfig } from '@/modules/scraps/entities/scrap-config.entity';
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
import {
extractDomain,
@@ -59,6 +60,7 @@ export class MailsService {
}
generateProductTableHTML(
+ keywords: { source: string; keywords: string }[],
products: ScrapItem[],
hotItems: { name: string }[],
): {
@@ -69,6 +71,24 @@ export class MailsService {
let hasHotItem = false;
+ const keywordHtml = keywords
+ .map((k) => {
+ return `
+
+ ${k.source}: ${k.keywords}
+
+ `;
+ })
+ .join('');
+
if (!products.length) {
return {
html: `
@@ -80,6 +100,27 @@ export class MailsService {
Product Listing
+
+
+ 🔎 Keywords matched today
+
+
+
+ ${keywordHtml}
+
+
Product Listing
No matching products found for your keywords today.
From: ${from}
@@ -160,6 +201,27 @@ export class MailsService {
${title}
+
+
+ 🔎 Keywords matched today
+
+
+
+ ${keywordHtml}
+
+
${title}
diff --git a/auto-bid-server/src/modules/scraps/services/tasks.service.ts b/auto-bid-server/src/modules/scraps/services/tasks.service.ts
index 21a4fee..44ad490 100644
--- a/auto-bid-server/src/modules/scraps/services/tasks.service.ts
+++ b/auto-bid-server/src/modules/scraps/services/tasks.service.ts
@@ -1,13 +1,16 @@
import { ConfigsService } from '@/modules/bids/services/configs.service';
import { DashboardService } from '@/modules/bids/services/dashboard.service';
import { MailsService } from '@/modules/mails/services/mails.service';
-import { delay } from '@/ultils';
+import { delay, extractDomainSmart } from '@/ultils';
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import * as moment from 'moment';
-import { Between } from 'typeorm';
+import { Between, In, Like, Not } from 'typeorm';
import { ScrapItemsService } from './scrap-item-config.service';
import { HotItemApi } from '@/modules/bids/apis/hot-item.api';
+import { ScrapConfigsService } from './scrap-config.service';
+import { capitalize } from 'lodash';
+import { ScrapItem } from '../entities/scrap-item.entity';
@Injectable()
export class TasksService {
@@ -15,20 +18,14 @@ export class TasksService {
constructor(
private readonly scrapItemsService: ScrapItemsService,
+ private readonly scrapConfigService: ScrapConfigsService,
private readonly mailsService: MailsService,
private readonly configsSerivce: ConfigsService,
private readonly dashboardService: DashboardService,
private readonly hotItemApi: HotItemApi,
) {}
- async runProcessAndSendReport(processName: string) {
- const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
- ?.value;
- if (!mails) {
- console.warn('No mails configured for report. Skipping.');
- return;
- }
-
+ async runProcess(processName: string) {
// Nếu process đang chạy, không chạy lại
const initialStatus =
await this.dashboardService.getStatusProcessByName(processName);
@@ -60,6 +57,15 @@ export class TasksService {
);
return;
}
+ }
+
+ async sendReportAllMail() {
+ const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
+ ?.value;
+ if (!mails) {
+ console.warn('No mails configured for report. Skipping.');
+ return;
+ }
// Khi process kết thúc => gửi mail
const startOfDay = new Date();
@@ -76,9 +82,23 @@ export class TasksService {
order: { updated_at: 'DESC' },
});
+ const scrapConfigs = await this.scrapConfigService.scrapConfigRepo.find({
+ relations: { web_bid: true },
+ });
+
+ const keywords = scrapConfigs.map((scrapConfig) => {
+ return {
+ source: capitalize(
+ extractDomainSmart(scrapConfig.web_bid.origin_url),
+ ),
+ keywords: scrapConfig.keywords,
+ };
+ });
+
const hotItems = await this.hotItemApi.listHotItem();
const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
+ keywords,
data,
hotItems,
);
@@ -95,10 +115,119 @@ export class TasksService {
}
}
+ async sendReportWithConfig({
+ subject,
+ data,
+ keywords,
+ }: {
+ subject: string;
+ keywords: { source: string; keywords: string }[];
+ data: ScrapItem[];
+ }) {
+ const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
+ ?.value;
+ if (!mails) {
+ console.warn('No mails configured for report. Skipping.');
+ return;
+ }
+
+ try {
+ const hotItems = await this.hotItemApi.listHotItem();
+
+ const { hasHotItem, html } = this.mailsService.generateProductTableHTML(
+ keywords,
+ data,
+ hotItems,
+ );
+
+ await this.mailsService.sendHtmlMailJob({
+ to: mails,
+ subject: `${hasHotItem ? '[HOT ITEMS] ' : ''} ${subject} – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`,
+ html: html,
+ });
+
+ console.log('Report mail sent successfully.');
+ } catch (err) {
+ console.error('Failed to generate or send report:', err);
+ }
+ }
+
// Thực hiện gọi scrape vào lúc 8h59 (giờ ÚC) mỗi ngày
- @Cron('59 5 * * *')
+ @Cron(CronExpression.EVERY_MINUTE)
+ // @Cron('59 5 * * *')
async handleScraps() {
const processName = 'scrape-data-keyword';
- await this.runProcessAndSendReport(processName);
+ // await this.runProcessAndSendReport(processName);
+
+ // await this.runProcess(processName);
+
+ const ramKeywords = ['DDR4', 'DDR5', 'DDR4|', 'DDR5|'];
+
+ // Khi process kết thúc => gửi mail
+ const startOfDay = new Date();
+ startOfDay.setHours(0, 0, 0, 0);
+ const endOfDay = new Date();
+ endOfDay.setHours(23, 59, 59, 999);
+
+ // Data Not Ram
+ const products = await this.scrapItemsService.scrapItemRepo.find({
+ where: {
+ updated_at: Between(startOfDay, endOfDay),
+ keyword: Not(In(ramKeywords)),
+ },
+ relations: { scrap_config: { web_bid: true } },
+ order: { updated_at: 'DESC' },
+ });
+
+ const qb = this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
+
+ ramKeywords.forEach((k, index) => {
+ qb.andWhere(`sc.keywords NOT LIKE :kw${index}`, {
+ [`kw${index}`]: `%${k}%`,
+ });
+ });
+
+ // Product Config
+ const productConfigs = await qb.getMany();
+
+ // Data Not Ram
+ const rams = await this.scrapItemsService.scrapItemRepo.find({
+ where: {
+ updated_at: Between(startOfDay, endOfDay),
+ keyword: In(ramKeywords),
+ },
+ relations: { scrap_config: { web_bid: true } },
+ order: { updated_at: 'DESC' },
+ });
+
+ // Ram Config
+ const ramConfigQb =
+ this.scrapConfigService.scrapConfigRepo.createQueryBuilder('sc');
+
+ ramKeywords.forEach((kw, i) => {
+ ramConfigQb.orWhere('sc.keywords LIKE :kw' + i, {
+ ['kw' + i]: `%${kw}%`,
+ });
+ });
+
+ const ramConfigs = await ramConfigQb.getMany();
+
+ await this.sendReportWithConfig({
+ data: products,
+ keywords: productConfigs.map((item) => ({
+ source: extractDomainSmart(item.search_url),
+ keywords: item.keywords,
+ })),
+ subject: 'Auction Items Matching Your Keywords',
+ });
+
+ await this.sendReportWithConfig({
+ data: rams,
+ keywords: ramConfigs.map((item) => ({
+ source: extractDomainSmart(item.search_url),
+ keywords: item.keywords,
+ })),
+ subject: 'DDR4, DDR5 Item Report',
+ });
}
}
diff --git a/scrape-data-keyword/models/pickles-scrap-model.js b/scrape-data-keyword/models/pickles-scrap-model.js
index 7874cf4..ce4d2e6 100644
--- a/scrape-data-keyword/models/pickles-scrap-model.js
+++ b/scrape-data-keyword/models/pickles-scrap-model.js
@@ -33,7 +33,7 @@ export class PicklesScrapModel extends ScrapModel {
const priceText = await newPage
.$eval("#bidPrefix > span.font-weight-bold", (el) =>
- el.textContent.trim()
+ el.textContent.trim(),
)
.catch(() => null);
@@ -50,7 +50,7 @@ export class PicklesScrapModel extends ScrapModel {
try {
const priceText = await elementHandle
.$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) =>
- el.textContent.trim()
+ el.textContent.trim(),
)
.catch(() => null);
@@ -61,48 +61,53 @@ export class PicklesScrapModel extends ScrapModel {
};
getItemsInHtml = async (data) => {
- await this.page.waitForSelector(
- ".content-wrapper_contentgridwrapper__3RCQZ > div.column",
- {
- timeout: 10000,
+ try {
+ await this.page.waitForSelector(
+ ".content-wrapper_contentgridwrapper__3RCQZ > div.column",
+ {
+ timeout: 10000,
+ },
+ );
+
+ const elements = await this.page.$$(
+ ".content-wrapper_contentgridwrapper__3RCQZ > div.column",
+ );
+
+ const results = [];
+
+ for (const el of elements) {
+ const url = await el
+ .$eval("main > a", (el) => el.getAttribute("href"))
+ .catch(() => null);
+
+ const image_url = await el
+ .$eval("div > div:first-child > div > div:first-child > img", (img) =>
+ img.getAttribute("src"),
+ )
+ .catch(() => null);
+
+ const name = await el
+ .$eval("header > h2:first-of-type", (el) => el.textContent.trim())
+ .catch(() => null);
+
+ const model = extractModelId(`${this.web_bid.origin_url}${url}`);
+
+ const current_price = await this.getPriceByEl(el, model);
+
+ results.push({
+ url: `${this.web_bid.origin_url}${url}`,
+ image_url,
+ name,
+ keyword: data.keyword,
+ model,
+ current_price: current_price,
+ scrap_config_id: this.scrap_config_id,
+ });
}
- );
- const elements = await this.page.$$(
- ".content-wrapper_contentgridwrapper__3RCQZ > div.column"
- );
- const results = [];
-
- for (const el of elements) {
- const url = await el
- .$eval("main > a", (el) => el.getAttribute("href"))
- .catch(() => null);
-
- const image_url = await el
- .$eval("div > div:first-child > div > div:first-child > img", (img) =>
- img.getAttribute("src")
- )
- .catch(() => null);
-
- const name = await el
- .$eval("header > h2:first-of-type", (el) => el.textContent.trim())
- .catch(() => null);
-
- const model = extractModelId(`${this.web_bid.origin_url}${url}`);
-
- const current_price = await this.getPriceByEl(el, model);
-
- results.push({
- url: `${this.web_bid.origin_url}${url}`,
- image_url,
- name,
- keyword: data.keyword,
- model,
- current_price: current_price,
- scrap_config_id: this.scrap_config_id,
- });
+ return results;
+ } catch (error) {
+ return [];
}
-
- return results;
};
}