diff --git a/auto-bid-admin/src/components/config/mails-config.tsx b/auto-bid-admin/src/components/config/mails-config.tsx index f10e059..af71bc2 100644 --- a/auto-bid-admin/src/components/config/mails-config.tsx +++ b/auto-bid-admin/src/components/config/mails-config.tsx @@ -117,7 +117,7 @@ export default function MailsConfig() { }; const handleAdd = async (mail: string) => { - const newMails = [...mails, mail]; + const newMails = [...mails, mail.trim()]; open(); const response = await upsertConfig({ diff --git a/auto-bid-server/bot-data/metadata.json b/auto-bid-server/bot-data/metadata.json index 712acf1..ed1b53a 100644 --- a/auto-bid-server/bot-data/metadata.json +++ b/auto-bid-server/bot-data/metadata.json @@ -1 +1 @@ -{"createdAt":1748224707802} \ No newline at end of file +{"createdAt":1748825131474} \ No newline at end of file diff --git a/auto-bid-server/src/modules/bids/apis/hot-item.api.ts b/auto-bid-server/src/modules/bids/apis/hot-item.api.ts new file mode 100644 index 0000000..45149c3 --- /dev/null +++ b/auto-bid-server/src/modules/bids/apis/hot-item.api.ts @@ -0,0 +1,31 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import axios from 'axios'; +import { BidsService } from '../services/bids.service'; + +@Injectable() +export class HotItemApi { + constructor( + private readonly bidsService: BidsService, + private readonly configService: ConfigService, + ) {} + + listHotItem = async () => { + try { + const res = await axios({ + method: 'GET', + baseURL: this.configService.get('NEW_ITEM_BASE_URL'), + url: '/disti/api/hotitem', + headers: { + // ...axios.defaults.headers.common, + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + this.configService.get('NEW_ITEM_TOKEN'), + }, + }); + + return res.data.data || []; + } catch (error) { + return []; + } + }; +} diff --git a/auto-bid-server/src/modules/bids/bids.module.ts b/auto-bid-server/src/modules/bids/bids.module.ts index 6cbecd2..8856d88 100644 --- a/auto-bid-server/src/modules/bids/bids.module.ts +++ b/auto-bid-server/src/modules/bids/bids.module.ts @@ -33,6 +33,7 @@ import { Config } from './entities/configs.entity'; import { AdminConfigsController } from './controllers/admin/admin-configs.controller'; import { BidMetadatasService } from './services/bid-metadatas.service'; import { BidMetadata } from './entities/bid-metadata.entity'; +import { HotItemApi } from './apis/hot-item.api'; @Module({ imports: [ @@ -76,6 +77,7 @@ import { BidMetadata } from './entities/bid-metadata.entity'; TasksService, ConfigsService, BidMetadatasService, + HotItemApi, ], exports: [ BotTelegramApi, @@ -83,6 +85,7 @@ import { BidMetadata } from './entities/bid-metadata.entity'; BidsService, ConfigsService, DashboardService, + HotItemApi, ], }) export class BidsModule {} diff --git a/auto-bid-server/src/modules/bids/services/mail/imap.service.ts b/auto-bid-server/src/modules/bids/services/mail/imap.service.ts index 555f628..98ed3fe 100644 --- a/auto-bid-server/src/modules/bids/services/mail/imap.service.ts +++ b/auto-bid-server/src/modules/bids/services/mail/imap.service.ts @@ -98,7 +98,8 @@ export class ImapService { const result = verifyCode(emailContent); if (!result) { - throw new Error('fetchLatestEmail: Name or Code is empty'); + console.log('fetchLatestEmail: Name or Code is empty'); + return; } const { code, name } = result; @@ -108,7 +109,8 @@ export class ImapService { }); if (!webBid) { - throw new Error('Not found web bid'); + console.log('Not found web bid'); + return; } // send message event diff --git a/auto-bid-server/src/modules/mails/services/mails.service.ts b/auto-bid-server/src/modules/mails/services/mails.service.ts index 9390c42..9e2a27c 100644 --- a/auto-bid-server/src/modules/mails/services/mails.service.ts +++ b/auto-bid-server/src/modules/mails/services/mails.service.ts @@ -1,15 +1,17 @@ -import { Injectable } from '@nestjs/common'; -import { MailerService } from '@nestjs-modules/mailer'; +import { Bid } from '@/modules/bids/entities/bid.entity'; import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; import { extractDomain, extractDomainSmart, formatEndTime, + isHotItemFn, isTimeReached, } from '@/ultils'; -import { Bid } from '@/modules/bids/entities/bid.entity'; +import { MailerService } from '@nestjs-modules/mailer'; import { InjectQueue } from '@nestjs/bull'; +import { Injectable } from '@nestjs/common'; import { Queue } from 'bull'; +import * as _ from 'lodash'; @Injectable() export class MailsService { @@ -52,17 +54,26 @@ export class MailsService { }); } - generateProductTableHTML(products: ScrapItem[]): string { + generateProductTableHTML( + products: ScrapItem[], + hotItems: { name: string }[], + ): { + html: string; + hasHotItem: boolean; + } { const from = process.env.MAIL_USER || 'no-reply@example.com'; + let hasHotItem = false; + if (!products.length) { - return ` + return { + html: ` - Products + Product Listing

Product Listing

@@ -70,33 +81,75 @@ export class MailsService {

From: ${from}

- `; + `, + hasHotItem, + }; } - const rows = products - .map( - (p) => ` - - Product Image - ${p.name} - ${p.current_price ? '$' + p.current_price : 'None'} - View - ${extractDomainSmart(p.scrap_config.web_bid.origin_url)} - - `, - ) + // Decorate products with isHotItem + const decorated = products.map((p) => { + const isHotItem = hotItems.some((obj) => + p.name.toLowerCase().includes(obj.name.toLowerCase()), + ); + if (isHotItem) hasHotItem = true; + + return { + ...p, + isHotItem, + isNew: + new Date(p.created_at).getTime() === new Date(p.updated_at).getTime(), + }; + }); + + // Separate hot and non-hot items, preserving original order + const hotItemsFirst = [ + ...decorated.filter((p) => p.isHotItem), + ...decorated.filter((p) => !p.isHotItem), + ]; + + const rows = hotItemsFirst + .map((p) => { + const isHotItem = isHotItemFn(p, hotItems); + + if (isHotItem) { + hasHotItem = true; + } + + const isNew = + new Date(p.created_at).getTime() === new Date(p.updated_at).getTime(); + + const nameStyle = isNew ? 'color: #2f9e44; font-weight: bold;' : ''; + const namePrefix = isHotItem + ? '[HOT ITEM] ' + : ''; + + return ` + + Product Image + ${namePrefix}${p.name} + ${p.current_price ? '$' + p.current_price : 'None'} + View + ${extractDomainSmart(p.scrap_config.web_bid.origin_url)} + + `; + }) .join(''); - return ` + const title = hasHotItem + ? '[HOT ITEMS] Product Listing' + : 'Product Listing'; + + return { + html: ` - Products + ${title} -

Product Listing

+

${title}

@@ -116,7 +169,9 @@ export class MailsService {

From: ${from}

- `; + `, + hasHotItem, + }; } getAuctionStatusEmailContent(bid: Bid): string { diff --git a/auto-bid-server/src/modules/scraps/services/tasks.service.ts b/auto-bid-server/src/modules/scraps/services/tasks.service.ts index d514679..bdae73e 100644 --- a/auto-bid-server/src/modules/scraps/services/tasks.service.ts +++ b/auto-bid-server/src/modules/scraps/services/tasks.service.ts @@ -3,10 +3,11 @@ import { DashboardService } from '@/modules/bids/services/dashboard.service'; import { MailsService } from '@/modules/mails/services/mails.service'; import { delay } from '@/ultils'; import { Injectable, Logger } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; +import { Cron, CronExpression } from '@nestjs/schedule'; import * as moment from 'moment'; import { Between } from 'typeorm'; import { ScrapItemsService } from './scrap-item-config.service'; +import { HotItemApi } from '@/modules/bids/apis/hot-item.api'; @Injectable() export class TasksService { @@ -17,6 +18,7 @@ export class TasksService { private readonly mailsService: MailsService, private readonly configsSerivce: ConfigsService, private readonly dashboardService: DashboardService, + private readonly hotItemApi: HotItemApi, ) {} async runProcessAndSendReport(processName: string) { @@ -71,13 +73,20 @@ export class TasksService { updated_at: Between(startOfDay, endOfDay), }, relations: { scrap_config: { web_bid: true } }, - order: { updated_at: 'DESC' }, + order: { updated_at: 'ASC' }, }); + const hotItems = await this.hotItemApi.listHotItem(); + + const { hasHotItem, html } = this.mailsService.generateProductTableHTML( + data, + hotItems, + ); + await this.mailsService.sendHtmlMailJob({ to: mails, - subject: `Auction Items Matching Your Keywords – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`, - html: this.mailsService.generateProductTableHTML(data), + subject: `${hasHotItem ? '[HOT ITEMS] ' : ''}Auction Items Matching Your Keywords – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`, + html: html, }); console.log('Report mail sent successfully.'); @@ -86,7 +95,8 @@ export class TasksService { } } - @Cron('58 5 * * *') + // @Cron('59 5 * * *') + // @Cron(CronExpression.EVERY_MINUTE) async handleScraps() { const processName = 'scrape-data-keyword'; await this.runProcessAndSendReport(processName); diff --git a/auto-bid-server/src/ultils/index.ts b/auto-bid-server/src/ultils/index.ts index 7d9e5d0..0d9577c 100644 --- a/auto-bid-server/src/ultils/index.ts +++ b/auto-bid-server/src/ultils/index.ts @@ -1,4 +1,5 @@ import { Bid } from '@/modules/bids/entities/bid.entity'; +import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; import * as moment from 'moment'; export function extractModelId(url: string): string | null { @@ -215,3 +216,9 @@ export function formatEndTime( export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const isHotItemFn = (item: ScrapItem, hotItems: any[]) => { + return hotItems.some((obj) => + item.name.toLowerCase().includes(obj.name.toLowerCase()), + ); +}; diff --git a/scrape-data-keyword/index.js b/scrape-data-keyword/index.js index 247c76d..8ca0a3c 100644 --- a/scrape-data-keyword/index.js +++ b/scrape-data-keyword/index.js @@ -22,6 +22,7 @@ const init = async () => { try { await model.action(); + console.log(`Loaded model [${model?.web_bid?.origin_url || ""}]`); for (const key of Object.keys(model.results)) { const dataArray = model.results[key]; const result = await upsertScapeItems(dataArray); diff --git a/scrape-data-keyword/models/allbids-scrap-model.js b/scrape-data-keyword/models/allbids-scrap-model.js index a7d11d4..b66ecd0 100644 --- a/scrape-data-keyword/models/allbids-scrap-model.js +++ b/scrape-data-keyword/models/allbids-scrap-model.js @@ -1,25 +1,12 @@ import axios from "axios"; -import browser from "../system/browser.js"; -import { extractModelId, extractNumber } from "../system/ultils.js"; +import { extractModelId } from "../system/ultils.js"; import { ScrapModel } from "./scrap-model.js"; export class AllbidsScrapModel extends ScrapModel { action = async () => { const urlsData = this.extractUrls(); - console.log({ urlsData }); - for (let item of urlsData) { - // await this.page.goto(item.url); - - // const data = await this.getItemsInHtml(item); - - // const results = this.filterItemByKeyword(item.keyword, data); - - // this.results[item.keyword] = results; - - // console.log({ results: this.results }); - const { data } = await axios({ url: item.url, method: "POST", @@ -37,14 +24,14 @@ export class AllbidsScrapModel extends ScrapModel { const auctions = data?.auctions || []; - const mappedData = auctions.map((item) => { + const mappedData = auctions.map((auction) => { return { - url: item.AucDetailsUrlLink, - image_url: item.aucThumbnailUrl, - name: item.aucTitle, - keyword: data.keyword, - model: extractModelId(item.AucDetailsUrlLink), - current_price: item.aucCurrentBid, + url: auction.AucDetailsUrlLink, + image_url: auction.aucThumbnailUrl, + name: auction.aucTitle, + keyword: item.keyword, + model: extractModelId(auction.AucDetailsUrlLink), + current_price: auction.aucCurrentBid, scrap_config_id: this.scrap_config_id, }; });