Deploy to staging #53
|
|
@ -1,178 +0,0 @@
|
|||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dutf-8">
|
||||
<title>Callcentric Email</title>
|
||||
<style type=3D"text/css" media=3D"all">
|
||||
body {
|
||||
font-family: arial,helvetica,sans-serif;
|
||||
font-size: 14px;
|
||||
line-height: 22px;
|
||||
color: #444;
|
||||
margin: 0;
|
||||
padding: 20px 0;
|
||||
background-color: #e5e5e5;
|
||||
outline: 0;
|
||||
vertical-align: top;
|
||||
}
|
||||
p {
|
||||
margin-bottom: 1.5em;
|
||||
}
|
||||
img {
|
||||
display: block;
|
||||
}
|
||||
a:link,
|
||||
a:visited {
|
||||
color: #006699;
|
||||
text-decoration: underline;
|
||||
margin: 0px;
|
||||
padding: 0px;
|
||||
}
|
||||
a:hover {=20
|
||||
text-decoration: none;
|
||||
}
|
||||
li {
|
||||
margin-bottom: 1em;
|
||||
line-height: 22px;
|
||||
}=09
|
||||
=2Efooter a {
|
||||
color: #aaaaaa;
|
||||
text-decoration: underline;
|
||||
}
|
||||
=2Efooter a:hover {
|
||||
text-decoration: none;
|
||||
}
|
||||
=2Eframe {
|
||||
width: 580px;
|
||||
}
|
||||
#btn {
|
||||
background: #3498db;
|
||||
background-image: -webkit-linear-gradient(top, #3498db, #2980b9);
|
||||
background-image: -moz-linear-gradient(top, #3498db, #2980b9);
|
||||
background-image: -ms-linear-gradient(top, #3498db, #2980b9);
|
||||
background-image: -o-linear-gradient(top, #3498db, #2980b9);
|
||||
background-image: linear-gradient(to bottom, #3498db, #2980b9);
|
||||
-webkit-border-radius: 28;
|
||||
-moz-border-radius: 28;
|
||||
border-radius: 28px;
|
||||
font-family: Arial;
|
||||
color: #ffffff;
|
||||
font-size: 20px;
|
||||
padding: 10px 20px 10px 20px;
|
||||
text-decoration: none;
|
||||
display: inline-block;
|
||||
}
|
||||
#btn:hover {
|
||||
background: #3cb0fd;
|
||||
background-image: -webkit-linear-gradient(top, #3cb0fd, #3498db);
|
||||
background-image: -moz-linear-gradient(top, #3cb0fd, #3498db);
|
||||
background-image: -ms-linear-gradient(top, #3cb0fd, #3498db);
|
||||
background-image: -o-linear-gradient(top, #3cb0fd, #3498db);
|
||||
background-image: linear-gradient(to bottom, #3cb0fd, #3498db);
|
||||
text-decoration: none;
|
||||
}
|
||||
=2Esms-box { font-family: arial,helvetica,sans-serif; font-size: 14px; padd=
|
||||
ing:12px; border:1px solid #e0e0e0; background-color: #e0e0e0; -webkit-bord=
|
||||
er-radius: 12px; -moz-border-radius: 12px; border-radius: 12px; word-wrap:b=
|
||||
reak-word; display:inline-block; }
|
||||
p { font-family: arial,helvetica,sans-serif; font-size: 14px; margin-bottom=
|
||||
: 1.5em; line-height: 22px; }
|
||||
|
||||
@media only screen and (max-device-width: 480px) {
|
||||
.frame {
|
||||
width: 442px !important;
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 22px !important;
|
||||
line-height: 28px !important;
|
||||
}
|
||||
ul li {
|
||||
font-size: 22px !important;
|
||||
line-height: 28px !important;
|
||||
}
|
||||
.section_header{
|
||||
font-size: 28px !important;
|
||||
line-height: 30px !important;
|
||||
}
|
||||
#btn {
|
||||
font-size: 28px !important;
|
||||
padding: 14px 24px 14px 24px !important;
|
||||
}
|
||||
}
|
||||
|
||||
</style>
|
||||
</head>
|
||||
<body style=3D"font-family: Arial, sans-serif; font-size: 14px; margin-bott=
|
||||
om: 1.5em; line-height: 22px; color: black; width: 100% !important; margin:=
|
||||
0; padding: 0px; background-color: #e5e5e5; outline: 0; vertical-align: to=
|
||||
p;" bgcolor=3D"#e5e5e5">
|
||||
<table width=3D"100%" cellpadding=3D"0" cellspacing=3D"0" border=3D"0"><tr>=
|
||||
<td style=3D"padding: 0px 0px;" align=3D"center">
|
||||
<table class=3D"head" width=3D"100%" bgcolor=3D"#c9c9c9" style=3D"backgroun=
|
||||
d-color:#c9c9c9;">
|
||||
<tr><td align=3D"center" style=3D"color:#f3f3f3; padding:5px; font-size:12=
|
||||
px;">Please do not respond to this automated message, you will not receive =
|
||||
a reply</td></tr>
|
||||
</table>
|
||||
</td></tr><tr><td style=3D"padding: 0px 10px;" align=3D"center">
|
||||
<table class=3D"frame" width=3D"580" cellpadding=3D"0" cellspacing=3D"0" bo=
|
||||
rder=3D"0" align=3D"center"><tbody>
|
||||
<tr><td>
|
||||
<table width=3D"100%" cellpadding=3D"0" cellspacing=3D"0" border=3D"0">
|
||||
<tr><td style=3D"padding: 30px 0px 10px 0px; color: #444; text-align: le=
|
||||
ft; vertical-align: middle;" align=3D"left"><a href=3D"https://www.callcent=
|
||||
ric.com"><img src=3D"https://www.callcentric.com/img/callcentric_logo_250_t=
|
||||
rans.png" width=3D"125" height=3D"33" alt=3D"Callcentric" border=3D"0"></a>=
|
||||
</td></tr>
|
||||
</table>
|
||||
</td></tr>
|
||||
<tr><td style=3D"background-color: #ffffff; padding: 0px 0px 50px; border-=
|
||||
radius: 5px 5px 0px 0px;">
|
||||
<table style=3D"padding-top: 40px;" width=3D"100%" cellpadding=3D"0" cell=
|
||||
spacing=3D"0" border=3D"0"><tbody>
|
||||
<tr><td class=3D"flex-column" style=3D"width: 40px;"></td>
|
||||
<td style=3D"text-align: left;" align=3D"left">
|
||||
<p>Dear Morgan Vu,</p><p>You have received an inbound text message from 223=
|
||||
95 on your DID 17146893350:</p><div class=3D"sms-box">Your LANGTONS verific=
|
||||
ation code is: 012152. This code will expire in 10 minutes.</div>
|
||||
|
||||
</td><td class=3D"flex-column" style=3D"width: 40px;"></td></tr>
|
||||
</tbody></table>
|
||||
</td></tr>
|
||||
<tr><td align=3D"left" class=3D"footer" style=3D"background-color: #f2f2f2=
|
||||
; color: #999999; padding: 0px 0px 40px; border-radius: 0px 0px 5px 5px;">
|
||||
<table border=3D"0" cellpadding=3D"0" cellspacing=3D"0" style=3D"padding-=
|
||||
top: 40px;" width=3D"100%"><tbody>
|
||||
<tr><td style=3D"width: 40px;"></td>
|
||||
<td align=3D"left" style=3D"text-align: left; font-family:Arial, Helvet=
|
||||
ica, sans-serif; font-size: 15px; line-height: 22px; color: #999999;">
|
||||
|
||||
Follow us on Twitter at:<br>
|
||||
<a href=3D"https://twitter.com/Callcentric">https://twitter.com/Callcentric=
|
||||
</a>
|
||||
<br><br>
|
||||
|
||||
If you have any questions/concerns, please contact our Customer Service Tea=
|
||||
m via the following page:<br>
|
||||
<a href=3D"https://www.callcentric.com/CustomerService">https://www.callcen=
|
||||
tric.com/CustomerService</a><br><br>
|
||||
|
||||
For instructions on how to unsubscribe from our Newsletters/Service Announc=
|
||||
ements please visit:<br>
|
||||
|
||||
<a href=3D"https://www.callcentric.com/faq/11/306">https://www.callcentric=
|
||||
=2Ecom/faq/11/306</a><br><br>
|
||||
|
||||
<a href=3D"https://www.callcentric.com">www.callcentric.com</a><br><br>
|
||||
|
||||
</td><td style=3D"width: 40px;"></td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</td></tr>
|
||||
<tr><td style=3D"padding: 20px 0; font-size: 15px; line-height: 25px; colo=
|
||||
r: #999999; text-align: center;" align=3D"center"></td></tr>
|
||||
</tbody></table></td></tr></table>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -25,6 +25,7 @@
|
|||
"migration:revert": "npm run typeorm -- -d ./src/config/typeorm.ts migration:revert"
|
||||
},
|
||||
"dependencies": {
|
||||
"@nestjs-modules/mailer": "^2.0.2",
|
||||
"@nestjs/common": "^10.0.0",
|
||||
"@nestjs/config": "^4.0.1",
|
||||
"@nestjs/core": "^10.0.0",
|
||||
|
|
@ -51,6 +52,7 @@
|
|||
"multer": "^1.4.5-lts.1",
|
||||
"mysql2": "^3.13.0",
|
||||
"nestjs-paginate": "^11.1.0",
|
||||
"nodemailer": "^7.0.3",
|
||||
"reflect-metadata": "^0.2.0",
|
||||
"rxjs": "^7.8.1",
|
||||
"sharp": "^0.33.5",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import { ClientAuthenticationMiddleware } from './modules/auth/middlewares/clien
|
|||
import { NotificationModule } from './modules/notification/notification.module';
|
||||
import { ScrapsModule } from './modules/scraps/scraps.module';
|
||||
import { ExtensionsModule } from './modules/extensions/extensions.module';
|
||||
import { MailsModule } from './modules/mails/mails.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
|
|
@ -28,6 +29,7 @@ import { ExtensionsModule } from './modules/extensions/extensions.module';
|
|||
NotificationModule,
|
||||
ScrapsModule,
|
||||
ExtensionsModule,
|
||||
MailsModule,
|
||||
],
|
||||
controllers: [],
|
||||
providers: [],
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ import { Config } from './entities/configs.entity';
|
|||
OutBidLog,
|
||||
WebBid,
|
||||
SendMessageHistory,
|
||||
Config
|
||||
Config,
|
||||
]),
|
||||
// AuthModule,
|
||||
AdminsModule,
|
||||
|
|
@ -69,8 +69,13 @@ import { Config } from './entities/configs.entity';
|
|||
ImapService,
|
||||
DashboardService,
|
||||
TasksService,
|
||||
ConfigsService
|
||||
ConfigsService,
|
||||
],
|
||||
exports: [
|
||||
BotTelegramApi,
|
||||
SendMessageHistoriesService,
|
||||
BidsService,
|
||||
ConfigsService,
|
||||
],
|
||||
exports: [BotTelegramApi, SendMessageHistoriesService, BidsService],
|
||||
})
|
||||
export class BidsModule {}
|
||||
|
|
|
|||
|
|
@ -8,22 +8,25 @@ import { Config } from '../entities/configs.entity';
|
|||
export class ConfigsService {
|
||||
public static CONFIG_KEYS = {
|
||||
REFRESH_TOOL_TIME: 'REFRESH_TOOL_TIME',
|
||||
MAIL_SCRAP_REPORT: 'MAIL_SCRAP_REPORT',
|
||||
};
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Config)
|
||||
readonly configRepo: Repository<Config>,
|
||||
private eventEmitter: EventEmitter2,
|
||||
) {}
|
||||
|
||||
async getConfig(key_name: keyof typeof ConfigsService.CONFIG_KEYS) {
|
||||
return await this.configRepo.findOne({ where: { key_name } }) || null;
|
||||
return (await this.configRepo.findOne({ where: { key_name } })) || null;
|
||||
}
|
||||
|
||||
async setConfig(key_name: keyof typeof ConfigsService.CONFIG_KEYS, value: string, type: 'string' | 'number') {
|
||||
return await this.configRepo.upsert(
|
||||
{ key_name, value , type},
|
||||
['key_name']
|
||||
);
|
||||
async setConfig(
|
||||
key_name: keyof typeof ConfigsService.CONFIG_KEYS,
|
||||
value: string,
|
||||
type: 'string' | 'number',
|
||||
) {
|
||||
return await this.configRepo.upsert({ key_name, value, type }, [
|
||||
'key_name',
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
import { MailerModule } from '@nestjs-modules/mailer';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { MailsService } from './services/mails.service';
|
||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
MailerModule.forRootAsync({
|
||||
imports: [ConfigModule],
|
||||
useFactory: async (config: ConfigService) => ({
|
||||
transport: {
|
||||
host: config.get<string>('MAIL_SERVER'),
|
||||
port: 465,
|
||||
secure: true, // true nếu port là 465
|
||||
auth: {
|
||||
user: config.get<string>('MAIL_USER'),
|
||||
pass: config.get<string>('MAIL_PASSWORD'),
|
||||
},
|
||||
},
|
||||
}),
|
||||
inject: [ConfigService],
|
||||
}),
|
||||
],
|
||||
providers: [MailsService],
|
||||
exports: [MailsService],
|
||||
})
|
||||
export class MailsModule {}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
import { Injectable } from '@nestjs/common';
|
||||
import { MailerService } from '@nestjs-modules/mailer';
|
||||
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
|
||||
import { extractDomain } from '@/ultils';
|
||||
|
||||
@Injectable()
|
||||
export class MailsService {
|
||||
constructor(private readonly mailerService: MailerService) {}
|
||||
|
||||
async sendPlainText(to: string, subject: string, content: string) {
|
||||
await this.mailerService.sendMail({
|
||||
to,
|
||||
subject,
|
||||
text: content,
|
||||
});
|
||||
}
|
||||
|
||||
async sendPlainHtml(to: string, subject: string, html: string) {
|
||||
const emails = to
|
||||
.split(',')
|
||||
.map((e) => e.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const mainRecipient = emails[0];
|
||||
const ccRecipients = emails.slice(1); // Những email còn lại
|
||||
|
||||
await this.mailerService.sendMail({
|
||||
to: mainRecipient,
|
||||
cc: ccRecipients.length > 0 ? ccRecipients : undefined,
|
||||
subject,
|
||||
html,
|
||||
});
|
||||
}
|
||||
|
||||
generateProductTableHTML(products: ScrapItem[]): string {
|
||||
const rows = products
|
||||
.map(
|
||||
(p) => `
|
||||
<tr>
|
||||
<td><img src="${p.image_url}" alt="Product Image" style="height: 40px; object-fit: contain; border-radius: 4px;" /></td>
|
||||
<td>${p.name}</td>
|
||||
<td style="font-weight: bold; color: #e03131;">$${p.current_price}</td>
|
||||
<td><a href="${p.url}" target="_blank" style="color: #007bff;">View</a></td>
|
||||
<td>${extractDomain(p.scrap_config.web_bid.origin_url)}</td>
|
||||
</tr>
|
||||
`,
|
||||
)
|
||||
.join('');
|
||||
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>Products</title>
|
||||
</head>
|
||||
<body style="font-family: sans-serif; background: #f8f9fa; padding: 20px;">
|
||||
<h2 style="text-align: center; color: #333;">Product Listing</h2>
|
||||
<div style="overflow-x: auto;">
|
||||
<table style="width: 100%; min-width: 500px; border-collapse: collapse; background: #fff; border-radius: 6px; box-shadow: 0 2px 6px rgba(0,0,0,0.05);">
|
||||
<thead style="background: #f1f3f5;">
|
||||
<tr>
|
||||
<th style="padding: 8px; text-align: left; border-bottom: 1px solid #ccc;">Image</th>
|
||||
<th style="padding: 8px; text-align: left; border-bottom: 1px solid #ccc;">Name</th>
|
||||
<th style="padding: 8px; text-align: left; border-bottom: 1px solid #ccc;">Price</th>
|
||||
<th style="padding: 8px; text-align: left; border-bottom: 1px solid #ccc;">Link</th>
|
||||
<th style="padding: 8px; text-align: left; border-bottom: 1px solid #ccc;">Domain</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${rows}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
async sendWithTemplate(to: string, subject: string, payload: any) {
|
||||
await this.mailerService.sendMail({
|
||||
to,
|
||||
subject,
|
||||
template: './welcome', // đường dẫn tương đối trong /templates
|
||||
context: payload, // dữ liệu cho template
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +1,21 @@
|
|||
import { MailsService } from '@/modules/mails/services/mails.service';
|
||||
import { Body, Controller, Get, Param, Post, Put } from '@nestjs/common';
|
||||
import { Between, IsNull, Not } from 'typeorm';
|
||||
import { CreateScrapConfigDto } from '../dto/scrap-config/create-scrap-config';
|
||||
import { ScrapConfigsService } from '../services/scrap-config.service';
|
||||
import { UpdateScrapConfigDto } from '../dto/scrap-config/update-scrap-config';
|
||||
import { ScrapConfig } from '../entities/scrap-config.entity';
|
||||
import { IsNull, Not } from 'typeorm';
|
||||
import { ScrapConfigsService } from '../services/scrap-config.service';
|
||||
import { ScrapItemsService } from '../services/scrap-item-config.service';
|
||||
import { ConfigsService } from '@/modules/bids/services/configs.service';
|
||||
import * as moment from 'moment';
|
||||
|
||||
@Controller('admin/scrap-configs')
|
||||
export class ScrapConfigsController {
|
||||
constructor(
|
||||
private readonly scrapConfigsService: ScrapConfigsService,
|
||||
private readonly scrapItemsService: ScrapItemsService,
|
||||
private readonly mailsService: MailsService,
|
||||
private readonly configsSerivce: ConfigsService,
|
||||
) {}
|
||||
|
||||
@Post()
|
||||
|
|
@ -28,6 +33,11 @@ export class ScrapConfigsController {
|
|||
|
||||
@Get()
|
||||
async test() {
|
||||
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
|
||||
.value;
|
||||
|
||||
if (!mails) return;
|
||||
|
||||
const scrapConfigs = await this.scrapConfigsService.scrapConfigRepo.find({
|
||||
where: {
|
||||
search_url: Not(IsNull()),
|
||||
|
|
@ -37,24 +47,39 @@ export class ScrapConfigsController {
|
|||
web_bid: true,
|
||||
},
|
||||
});
|
||||
|
||||
const models = this.scrapConfigsService.scrapModels(scrapConfigs);
|
||||
|
||||
await Promise.allSettled(
|
||||
models.map(async (item) => {
|
||||
await item.action();
|
||||
|
||||
Object.keys(item.results).forEach(async (key) => {
|
||||
const data = item.results[key];
|
||||
|
||||
await this.scrapItemsService.scrapItemRepo.upsert(data, [
|
||||
'model',
|
||||
'scrap_config',
|
||||
]);
|
||||
});
|
||||
for (const key of Object.keys(item.results)) {
|
||||
const dataArray = item.results[key];
|
||||
const result =
|
||||
await this.scrapItemsService.upsertScrapItems(dataArray);
|
||||
console.log(result);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const startOfDay = new Date();
|
||||
startOfDay.setHours(0, 0, 0, 0);
|
||||
|
||||
const endOfDay = new Date();
|
||||
endOfDay.setHours(23, 59, 59, 999);
|
||||
|
||||
const data = await this.scrapItemsService.scrapItemRepo.find({
|
||||
where: {
|
||||
updated_at: Between(startOfDay, endOfDay),
|
||||
},
|
||||
relations: { scrap_config: { web_bid: true } },
|
||||
order: { updated_at: 'DESC' },
|
||||
});
|
||||
|
||||
await this.mailsService.sendPlainHtml(
|
||||
mails,
|
||||
`Auction Items Matching Your Keywords – Daily Update ${moment(new Date()).format('YYYY-MM-DD HH:mm')}`,
|
||||
this.mailsService.generateProductTableHTML(data),
|
||||
);
|
||||
|
||||
return { a: 'abc' };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import {
|
||||
Column,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
Unique,
|
||||
|
|
@ -9,7 +10,7 @@ import { ScrapConfig } from './scrap-config.entity';
|
|||
import { Timestamp } from './timestamp';
|
||||
|
||||
@Entity('scrap-items')
|
||||
@Unique(['model', 'scrap_config'])
|
||||
@Unique(['model', 'scrap_config_id'])
|
||||
export class ScrapItem extends Timestamp {
|
||||
@PrimaryGeneratedColumn('increment')
|
||||
id: number;
|
||||
|
|
@ -29,11 +30,15 @@ export class ScrapItem extends Timestamp {
|
|||
@Column({ nullable: true, default: null })
|
||||
keyword: string;
|
||||
|
||||
@Column({ nullable: true, default: null })
|
||||
@Column({ nullable: true, default: null, type: 'float' })
|
||||
current_price: number;
|
||||
|
||||
@Column()
|
||||
scrap_config_id: number;
|
||||
|
||||
@ManyToOne(() => ScrapConfig, (web) => web.scrap_items, {
|
||||
onDelete: 'CASCADE',
|
||||
})
|
||||
@JoinColumn({ name: 'scrap_config_id' })
|
||||
scrap_config: ScrapConfig;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import axios from 'axios';
|
||||
import { ScrapInterface } from '../../scrap-interface';
|
||||
import { ScrapModel } from '../../scrap-model';
|
||||
import { ScrapInterface } from '../scrap-interface';
|
||||
import { ScrapModel } from '../scrap-model';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { Element } from 'domhandler';
|
||||
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
|
||||
|
|
@ -9,7 +9,6 @@ export class GraysScrapModel extends ScrapModel {
|
|||
action = async () => {
|
||||
const urls = this.extractUrls();
|
||||
|
||||
console.log({ urls });
|
||||
const results = await Promise.allSettled(
|
||||
urls.map(async (item) => ({
|
||||
html: (await axios.get(item.url)).data,
|
||||
|
|
@ -37,6 +36,21 @@ export class GraysScrapModel extends ScrapModel {
|
|||
);
|
||||
};
|
||||
|
||||
getPriceByEl = ($: cheerio.CheerioAPI, el: Element): number | null => {
|
||||
const selectors = [
|
||||
'.sc-ijDOKB.sc-bStcSt.ikmQUw.eEycyP', // Single product price
|
||||
'.sc-ijDOKB.ikmQUw', // Multiple product price
|
||||
];
|
||||
|
||||
for (const selector of selectors) {
|
||||
const text = $(el).find(selector).text();
|
||||
const price = extractNumber(text);
|
||||
if (price) return price;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
getItemsInHtml = async ({
|
||||
html,
|
||||
keyword,
|
||||
|
|
@ -59,10 +73,8 @@ export class GraysScrapModel extends ScrapModel {
|
|||
model: extractModelId(url),
|
||||
keyword,
|
||||
url,
|
||||
current_price: extractNumber(
|
||||
$(el).find('.sc-ijDOKB.sc-bStcSt.ikmQUw.eEycyP').text(),
|
||||
),
|
||||
scrap_config: { id: this.scrap_config_id },
|
||||
current_price: this.getPriceByEl($, el),
|
||||
scrap_config_id: this.scrap_config_id,
|
||||
} as ScrapItem;
|
||||
});
|
||||
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity';
|
||||
import { extractModelId, extractNumber } from '@/ultils';
|
||||
import axios from 'axios';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { Element } from 'domhandler';
|
||||
import { ScrapModel } from '../scrap-model';
|
||||
|
||||
export class LangtonsScrapModel extends ScrapModel {
|
||||
action = async () => {
|
||||
const urls = this.extractUrls();
|
||||
|
||||
console.log({ urls });
|
||||
const results = await Promise.allSettled(
|
||||
urls.map(async (item) => ({
|
||||
html: (
|
||||
await axios.get(item.url, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html',
|
||||
},
|
||||
maxRedirects: 5, // default là 5, bạn có thể tăng lên
|
||||
})
|
||||
).data,
|
||||
keyword: item.keyword,
|
||||
})),
|
||||
);
|
||||
|
||||
console.log(
|
||||
results.map((r) => ({
|
||||
status: r.status,
|
||||
reason:
|
||||
r.status === 'rejected'
|
||||
? (r as PromiseRejectedResult).reason.message
|
||||
: undefined,
|
||||
})),
|
||||
);
|
||||
|
||||
const htmlsData = results
|
||||
.filter((res) => res.status === 'fulfilled')
|
||||
.map((res) => (res as PromiseFulfilledResult<any>).value);
|
||||
|
||||
console.log({ htmlsData });
|
||||
|
||||
await Promise.all(
|
||||
htmlsData.map(async (cur) => {
|
||||
try {
|
||||
const data = await this.getItemsInHtml(cur);
|
||||
const results = this.filterItemByKeyword(cur.keyword, data);
|
||||
|
||||
this.results[cur.keyword] = results; // hoặc push như gợi ý trên
|
||||
return results;
|
||||
} catch (err) {
|
||||
console.error(`❌ Error with keyword ${cur.keyword}:`, err);
|
||||
return []; // fallback
|
||||
}
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
getPriceByEl = ($: cheerio.CheerioAPI, el: Element): number | null => {
|
||||
const selectors = [
|
||||
'.sc-ijDOKB.sc-bStcSt.ikmQUw.eEycyP', // Single product price
|
||||
'.sc-ijDOKB.ikmQUw', // Multiple product price
|
||||
];
|
||||
|
||||
for (const selector of selectors) {
|
||||
const text = $(el).find(selector).text();
|
||||
const price = extractNumber(text);
|
||||
if (price) return price;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
getItemsInHtml = async ({
|
||||
html,
|
||||
keyword,
|
||||
}: {
|
||||
html: string;
|
||||
keyword: string;
|
||||
}) => {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const container = $('.row.product-grid.grid-view');
|
||||
|
||||
const items = container.children('div').toArray();
|
||||
|
||||
const results = items.map((el) => {
|
||||
const url = $(el).find('a.js-pdp-link.pdp-link-anchor').attr('href');
|
||||
|
||||
return {
|
||||
name: $(el).find('.link.js-pdp-link').text().trim(),
|
||||
image_url: $(el).find('img.tile-image.loaded').attr('src'),
|
||||
model: extractModelId(url),
|
||||
keyword,
|
||||
url,
|
||||
current_price: extractNumber(
|
||||
$(el).find('div.max-bid-price.price').text(),
|
||||
),
|
||||
scrap_config_id: this.scrap_config_id,
|
||||
} as ScrapItem;
|
||||
});
|
||||
|
||||
console.log({ results });
|
||||
return results;
|
||||
};
|
||||
}
|
||||
|
|
@ -6,9 +6,15 @@ import { ScrapConfigsService } from './services/scrap-config.service';
|
|||
import { ScrapConfigsController } from './controllers/scrap-config.controller';
|
||||
import { TasksService } from './services/tasks.service';
|
||||
import { ScrapItemsService } from './services/scrap-item-config.service';
|
||||
import { MailsModule } from '../mails/mails.module';
|
||||
import { BidsModule } from '../bids/bids.module';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([ScrapConfig, ScrapItem])],
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([ScrapConfig, ScrapItem]),
|
||||
MailsModule,
|
||||
BidsModule,
|
||||
],
|
||||
providers: [ScrapConfigsService, TasksService, ScrapItemsService],
|
||||
exports: [ScrapConfigsService, TasksService, ScrapItemsService],
|
||||
controllers: [ScrapConfigsController],
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ import { CreateScrapConfigDto } from '../dto/scrap-config/create-scrap-config';
|
|||
import { UpdateScrapConfigDto } from '../dto/scrap-config/update-scrap-config';
|
||||
import axios from 'axios';
|
||||
import { WebBid } from '@/modules/bids/entities/wed-bid.entity';
|
||||
import { GraysScrapModel } from '../models/https:/www.grays.com/grays-scrap-model';
|
||||
import { GraysScrapModel } from '../models/www.grays.com/grays-scrap-model';
|
||||
import { LangtonsScrapModel } from '../models/www.langtons.com.au/langtons-scrap-model';
|
||||
|
||||
@Injectable()
|
||||
export class ScrapConfigsService {
|
||||
|
|
|
|||
|
|
@ -9,4 +9,53 @@ export class ScrapItemsService {
|
|||
@InjectRepository(ScrapItem)
|
||||
readonly scrapItemRepo: Repository<ScrapItem>,
|
||||
) {}
|
||||
|
||||
async upsertScrapItems(items: ScrapItem[]) {
|
||||
if (!items.length) return { inserted: 0, updated: 0 };
|
||||
|
||||
// Lấy keys để query bản ghi đã tồn tại
|
||||
const keys = items.map((d) => ({
|
||||
model: d.model,
|
||||
scrap_config_id: d.scrap_config_id,
|
||||
}));
|
||||
|
||||
// Tìm các bản ghi đã có trong DB
|
||||
const existingItems = await this.scrapItemRepo.findBy(keys);
|
||||
|
||||
const toUpdate = [];
|
||||
const toInsert = [];
|
||||
|
||||
items.forEach((itemData) => {
|
||||
const exist = existingItems.find(
|
||||
(e) =>
|
||||
e.model === itemData.model &&
|
||||
e.scrap_config_id === itemData.scrap_config_id,
|
||||
);
|
||||
|
||||
if (exist) {
|
||||
toUpdate.push({
|
||||
...exist,
|
||||
...itemData,
|
||||
updated_at: new Date(),
|
||||
});
|
||||
} else {
|
||||
toInsert.push(itemData);
|
||||
}
|
||||
});
|
||||
|
||||
// Thực hiện insert
|
||||
if (toInsert.length) {
|
||||
await this.scrapItemRepo.insert(toInsert);
|
||||
}
|
||||
|
||||
// Thực hiện update
|
||||
if (toUpdate.length) {
|
||||
await this.scrapItemRepo.save(toUpdate);
|
||||
}
|
||||
|
||||
return {
|
||||
inserted: toInsert.length,
|
||||
updated: toUpdate.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,72 @@
|
|||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { IsNull, Not } from 'typeorm';
|
||||
import { Between, IsNull, Not } from 'typeorm';
|
||||
import { ScrapConfigsService } from './scrap-config.service';
|
||||
import { ScrapItemsService } from './scrap-item-config.service';
|
||||
import { MailsService } from '@/modules/mails/services/mails.service';
|
||||
import { ConfigsService } from '@/modules/bids/services/configs.service';
|
||||
import * as moment from 'moment';
|
||||
|
||||
@Injectable()
|
||||
export class TasksService {
|
||||
private readonly logger = new Logger(TasksService.name);
|
||||
|
||||
constructor(private readonly scrapConfigsService: ScrapConfigsService) {}
|
||||
constructor(
|
||||
private readonly scrapConfigsService: ScrapConfigsService,
|
||||
private readonly scrapItemsService: ScrapItemsService,
|
||||
private readonly mailsService: MailsService,
|
||||
private readonly configsSerivce: ConfigsService,
|
||||
) {}
|
||||
|
||||
@Cron(CronExpression.EVERY_MINUTE)
|
||||
@Cron('0 2 * * *')
|
||||
async handleScraps() {
|
||||
// const scrapConfigs = await this.scrapConfigsService.scrapConfigRepo.find({
|
||||
// where: {
|
||||
// search_url: Not(IsNull()),
|
||||
// keywords: Not(IsNull()),
|
||||
// },
|
||||
// relations: {
|
||||
// web_bid: true,
|
||||
// },
|
||||
// });
|
||||
// const models = this.scrapConfigsService.scrapModels(scrapConfigs);
|
||||
// await Promise.allSettled(models.map(async (item) => await item.action()));
|
||||
const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT'))
|
||||
.value;
|
||||
|
||||
if (!mails) return;
|
||||
|
||||
const scrapConfigs = await this.scrapConfigsService.scrapConfigRepo.find({
|
||||
where: {
|
||||
search_url: Not(IsNull()),
|
||||
keywords: Not(IsNull()),
|
||||
},
|
||||
relations: {
|
||||
web_bid: true,
|
||||
},
|
||||
});
|
||||
const models = this.scrapConfigsService.scrapModels(scrapConfigs);
|
||||
await Promise.allSettled(
|
||||
models.map(async (item) => {
|
||||
await item.action();
|
||||
for (const key of Object.keys(item.results)) {
|
||||
const dataArray = item.results[key];
|
||||
const result =
|
||||
await this.scrapItemsService.upsertScrapItems(dataArray);
|
||||
console.log(result);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const startOfDay = new Date();
|
||||
startOfDay.setHours(0, 0, 0, 0);
|
||||
|
||||
const endOfDay = new Date();
|
||||
endOfDay.setHours(23, 59, 59, 999);
|
||||
|
||||
const data = await this.scrapItemsService.scrapItemRepo.find({
|
||||
where: {
|
||||
updated_at: Between(startOfDay, endOfDay),
|
||||
},
|
||||
relations: { scrap_config: { web_bid: true } },
|
||||
order: { updated_at: 'DESC' },
|
||||
});
|
||||
|
||||
await this.mailsService.sendPlainHtml(
|
||||
mails,
|
||||
`Auction Items Matching Your Keywords – Daily Update ${moment(new Date()).format('YYYY-MM-DD HH:mm')}`,
|
||||
this.mailsService.generateProductTableHTML(data),
|
||||
);
|
||||
|
||||
console.log('Send report success');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue