diff --git a/.DS_Store b/.DS_Store index 4737d85..64dbd43 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/auto-bid-admin/src/apis/config.ts b/auto-bid-admin/src/apis/config.ts new file mode 100644 index 0000000..599478e --- /dev/null +++ b/auto-bid-admin/src/apis/config.ts @@ -0,0 +1,33 @@ +import { handleError } from "."; +import axios from "../lib/axios"; +import { IConfig } from "../system/type"; +import { removeFalsyValues } from "../utils"; + +export const CONFIG_KEYS = { + MAIL_SCRAP_REPORT: "MAIL_SCRAP_REPORT", +}; + +export const getConfig = async (key: keyof typeof CONFIG_KEYS) => { + return await axios({ + url: "configs/" + key, + withCredentials: true, + method: "GET", + }); +}; + +export const upsertConfig = async (data: IConfig) => { + const { key_name, value, type } = removeFalsyValues(data, ["value"]); + + try { + const { data } = await axios({ + url: "configs/upsert", + withCredentials: true, + method: "POST", + data: { key_name, value: value || null, type }, + }); + + return data; + } catch (error) { + handleError(error); + } +}; diff --git a/auto-bid-admin/src/components/config/mails-config.tsx b/auto-bid-admin/src/components/config/mails-config.tsx new file mode 100644 index 0000000..f10e059 --- /dev/null +++ b/auto-bid-admin/src/components/config/mails-config.tsx @@ -0,0 +1,161 @@ +import { + ActionIcon, + Box, + Card, + Group, + LoadingOverlay, + Text, + TextInput, +} from "@mantine/core"; +import { useForm, zodResolver } from "@mantine/form"; +import { IconAt, IconMinus, IconPlus } from "@tabler/icons-react"; +import { useEffect, useMemo, useState } from "react"; +import { z } from "zod"; +import { getConfig, upsertConfig } from "../../apis/config"; +import { IConfig } from "../../system/type"; +import { useConfirmStore } from "../../lib/zustand/use-confirm"; +import { useDisclosure } from "@mantine/hooks"; + +const schema = z.object({ + email: z + .string({ message: "Email is required" }) + .email({ message: "Invalid email address" }), +}); + +const MailInput = ({ + initValue, + onDelete, + onAdd, +}: { + initValue?: string; + onDelete?: (data: string) => void; + onAdd?: (data: string) => Promise; +}) => { + const form = useForm({ + initialValues: { + email: initValue || "", + }, + validate: zodResolver(schema), + }); + + return ( +
{ + await onAdd(values.email); + form.reset(); + } + : () => {} + )} + className="flex items-start gap-2 w-full" + > + } + placeholder="Enter email" + className="flex-1" + size="xs" + /> + onDelete(initValue) : undefined} + type={!initValue ? "submit" : "button"} + color={initValue ? "red" : "blue"} + variant="light" + > + {initValue ? : } + + + ); +}; + +export default function MailsConfig() { + const [config, setConfig] = useState(null); + const { setConfirm } = useConfirmStore(); + const [opened, { open, close }] = useDisclosure(false); + useEffect(() => { + fetchConfig(); + }, []); + + const mails = useMemo(() => { + if (!config) return []; + + return config?.value?.split(", ").length > 0 + ? config?.value.split(",") + : []; + }, [config]); + + const fetchConfig = async () => { + const response = await getConfig("MAIL_SCRAP_REPORT"); + + if (!response || ![200, 201].includes(response.data?.status_code)) return; + + setConfig(response.data.data); + }; + + const handleDelete = (mail: string) => { + setConfirm({ + message: "Are you want to delete: " + mail, + title: "Delete", + handleOk: async () => { + open(); + const newMails = mails.filter((item) => item !== mail); + + if (!config) return; + + const response = await upsertConfig({ + ...(config as IConfig), + value: newMails.join(", "), + }); + + if (response) { + fetchConfig(); + } + close(); + }, + }); + }; + + const handleAdd = async (mail: string) => { + const newMails = [...mails, mail]; + + open(); + const response = await upsertConfig({ + ...(config as IConfig), + value: newMails.join(", "), + }); + + if (response) { + fetchConfig(); + } + close(); + }; + + return ( + + + + Mails + + + + + + {mails.length > 0 && + mails.map((mail) => { + return ( + + ); + })} + + + + + + + ); +} diff --git a/auto-bid-admin/src/components/user-menu.tsx b/auto-bid-admin/src/components/user-menu.tsx index b986611..ad4895d 100644 --- a/auto-bid-admin/src/components/user-menu.tsx +++ b/auto-bid-admin/src/components/user-menu.tsx @@ -1,122 +1,180 @@ -import { Avatar, Button, LoadingOverlay, Menu, Modal, PasswordInput } from '@mantine/core'; -import { useForm, zodResolver } from '@mantine/form'; -import { useDisclosure } from '@mantine/hooks'; -import { IconKey, IconLogout, IconSettings, IconUser } from '@tabler/icons-react'; -import { useState } from 'react'; -import { useNavigate } from 'react-router'; -import { Link } from 'react-router-dom'; -import { z } from 'zod'; -import { changePassword, logout } from '../apis/auth'; -import { useConfirmStore } from '../lib/zustand/use-confirm'; -import Links from '../system/links'; +import { + Avatar, + Button, + LoadingOverlay, + Menu, + Modal, + PasswordInput, +} from "@mantine/core"; +import { useForm, zodResolver } from "@mantine/form"; +import { useDisclosure } from "@mantine/hooks"; +import { + IconCode, + IconKey, + IconLogout, + IconSettings, + IconUser, +} from "@tabler/icons-react"; +import { useState } from "react"; +import { useNavigate } from "react-router"; +import { Link } from "react-router-dom"; +import { z } from "zod"; +import { changePassword, logout } from "../apis/auth"; +import { useConfirmStore } from "../lib/zustand/use-confirm"; +import Links from "../system/links"; const schema = z - .object({ - currentPassword: z.string().min(6, 'Current password must be at least 6 characters'), - newPassword: z.string().min(6, 'New password must be at least 6 characters'), - confirmPassword: z.string(), - }) - .refine((data) => data.newPassword === data.confirmPassword, { - path: ['confirmPassword'], - message: 'Passwords do not match', - }); + .object({ + currentPassword: z + .string() + .min(6, "Current password must be at least 6 characters"), + newPassword: z + .string() + .min(6, "New password must be at least 6 characters"), + confirmPassword: z.string(), + }) + .refine((data) => data.newPassword === data.confirmPassword, { + path: ["confirmPassword"], + message: "Passwords do not match", + }); export default function UserMenu() { - const [opened, { open, close }] = useDisclosure(false); + const [opened, { open, close }] = useDisclosure(false); - const { setConfirm } = useConfirmStore(); + const { setConfirm } = useConfirmStore(); - const [loading, setLoading] = useState(false); + const [loading, setLoading] = useState(false); - const navigate = useNavigate(); - const form = useForm({ - initialValues: { - currentPassword: '', - newPassword: '', - confirmPassword: '', - }, - validate: zodResolver(schema), + const navigate = useNavigate(); + const form = useForm({ + initialValues: { + currentPassword: "", + newPassword: "", + confirmPassword: "", + }, + validate: zodResolver(schema), + }); + + const handleSubmit = async (values: typeof form.values) => { + await handleChangePassword(values); + }; + + const handleLogout = async () => { + setConfirm({ + title: "Are you wan't to logout?", + message: "This account will logout !", + okButton: { value: "Logout" }, + handleOk: async () => { + const data = await logout(); + + if (data && data.data) { + navigate(Links.LOGIN); + } + }, }); + }; - const handleSubmit = async (values: typeof form.values) => { - await handleChangePassword(values); - }; - - const handleLogout = async () => { - setConfirm({ - title: "Are you wan't to logout?", - message: 'This account will logout !', - okButton: { value: 'Logout' }, - handleOk: async () => { - const data = await logout(); - - if (data && data.data) { - navigate(Links.LOGIN); - } - }, + const handleChangePassword = async (values: typeof form.values) => { + setConfirm({ + title: "Are you wan't to update password", + message: "This account will change password !", + okButton: { value: "Sure" }, + handleOk: async () => { + setLoading(true); + const data = await changePassword({ + newPassword: values.newPassword, + password: values.currentPassword, }); - }; - const handleChangePassword = async (values: typeof form.values) => { - setConfirm({ - title: "Are you wan't to update password", - message: 'This account will change password !', - okButton: { value: 'Sure' }, - handleOk: async () => { - setLoading(true); - const data = await changePassword({ - newPassword: values.newPassword, - password: values.currentPassword, - }); + setLoading(false); - setLoading(false); + if (data && data.data) { + navigate(Links.LOGIN); + close(); + } + }, + }); + }; - if (data && data.data) { - navigate(Links.LOGIN); - close(); - } - }, - }); - }; + return ( + <> + + + + + + - return ( - <> - - - - - - + + Account + }> + Change password + + } + > + Keys + - - Account - }> - Change password - - }> - Keys - + } + > + Configs + - + - }> - Logout - - - + } + > + Logout + + + - -
- - - - - + +
+ + + + + - -
- - ); + +
+ + ); } diff --git a/auto-bid-admin/src/pages/configs.tsx b/auto-bid-admin/src/pages/configs.tsx new file mode 100644 index 0000000..cb68288 --- /dev/null +++ b/auto-bid-admin/src/pages/configs.tsx @@ -0,0 +1,10 @@ +import { Box } from "@mantine/core"; +import MailsConfig from "../components/config/mails-config"; + +export default function Configs() { + return ( + + + + ); +} diff --git a/auto-bid-admin/src/pages/index.ts b/auto-bid-admin/src/pages/index.ts index 72ac18b..4995328 100644 --- a/auto-bid-admin/src/pages/index.ts +++ b/auto-bid-admin/src/pages/index.ts @@ -1,5 +1,6 @@ -export { default as Dashboard } from './dashboard'; -export { default as Bids } from './bids'; -export { default as OutBidsLog } from './out-bids-log'; -export { default as Login } from './login'; -export { default as App } from './app'; +export { default as Dashboard } from "./dashboard"; +export { default as Bids } from "./bids"; +export { default as OutBidsLog } from "./out-bids-log"; +export { default as Login } from "./login"; +export { default as App } from "./app"; +export { default as Configs } from "./configs"; diff --git a/auto-bid-admin/src/system/links.ts b/auto-bid-admin/src/system/links.ts index 649831c..eb80249 100644 --- a/auto-bid-admin/src/system/links.ts +++ b/auto-bid-admin/src/system/links.ts @@ -1,70 +1,87 @@ -import { IconHammer, IconHome2, IconKey, IconMessage, IconOutlet, IconPageBreak, IconUserCheck } from '@tabler/icons-react'; -import { Bids, Dashboard, OutBidsLog } from '../pages'; -import WebBids from '../pages/web-bids'; -import SendMessageHistories from '../pages/send-message-histories'; -import Admins from '../pages/admins'; -import GenerateKeys from '../pages/generate-keys'; +import { + IconHammer, + IconHome2, + IconKey, + IconMessage, + IconOutlet, + IconPageBreak, + IconSettings, + IconUserCheck, +} from "@tabler/icons-react"; +import { Bids, Configs, Dashboard, OutBidsLog } from "../pages"; +import WebBids from "../pages/web-bids"; +import SendMessageHistories from "../pages/send-message-histories"; +import Admins from "../pages/admins"; +import GenerateKeys from "../pages/generate-keys"; export default class Links { - public static DASHBOARD = '/dashboard'; - public static BIDS = '/bids'; - public static WEBS = '/webs'; - public static OUT_BIDS_LOG = '/out-bids-log'; - public static SEND_MESSAGE_HISTORIES = '/send-message-histories'; - public static GENERATE_KEYS = '/generate-keys'; - public static ADMINS = '/admins'; + public static DASHBOARD = "/dashboard"; + public static BIDS = "/bids"; + public static WEBS = "/webs"; + public static OUT_BIDS_LOG = "/out-bids-log"; + public static SEND_MESSAGE_HISTORIES = "/send-message-histories"; + public static GENERATE_KEYS = "/generate-keys"; + public static ADMINS = "/admins"; + public static CONFIGS = "/configs"; - public static HOME = '/'; - public static LOGIN = '/login'; + public static HOME = "/"; + public static LOGIN = "/login"; - public static MENUS = [ - { - path: this.DASHBOARD, - title: 'Dashboard', - icon: IconHome2, - element: Dashboard, - show: true, - }, - { - path: this.ADMINS, - title: 'Admins', - icon: IconUserCheck, - element: Admins, - show: true, - }, - { - path: this.WEBS, - title: 'Webs', - icon: IconPageBreak, - element: WebBids, - show: true, - }, - { - path: this.BIDS, - title: 'Bids', - icon: IconHammer, - element: Bids, - show: true, - }, - { - path: this.OUT_BIDS_LOG, - title: 'Out bids log', - icon: IconOutlet, - element: OutBidsLog, - show: true, - }, - { - path: this.SEND_MESSAGE_HISTORIES, - title: 'Send message histories', - icon: IconMessage, - element: SendMessageHistories, - show: true, - }, - { - path: this.GENERATE_KEYS, - title: 'Generate keys', - icon: IconKey, - element: GenerateKeys, - show: false, - }, - ]; + public static MENUS = [ + { + path: this.DASHBOARD, + title: "Dashboard", + icon: IconHome2, + element: Dashboard, + show: true, + }, + { + path: this.ADMINS, + title: "Admins", + icon: IconUserCheck, + element: Admins, + show: true, + }, + { + path: this.WEBS, + title: "Webs", + icon: IconPageBreak, + element: WebBids, + show: true, + }, + { + path: this.BIDS, + title: "Bids", + icon: IconHammer, + element: Bids, + show: true, + }, + { + path: this.OUT_BIDS_LOG, + title: "Out bids log", + icon: IconOutlet, + element: OutBidsLog, + show: true, + }, + { + path: this.SEND_MESSAGE_HISTORIES, + title: "Send message histories", + icon: IconMessage, + element: SendMessageHistories, + show: true, + }, + { + path: this.GENERATE_KEYS, + title: "Generate keys", + icon: IconKey, + element: GenerateKeys, + show: false, + }, + { + path: this.CONFIGS, + title: "Configs", + icon: IconSettings, + element: Configs, + show: false, + }, + ]; } diff --git a/auto-bid-admin/src/system/type/index.ts b/auto-bid-admin/src/system/type/index.ts index 0f5b0a3..0d23dd0 100644 --- a/auto-bid-admin/src/system/type/index.ts +++ b/auto-bid-admin/src/system/type/index.ts @@ -82,6 +82,13 @@ export interface IBid extends ITimestamp { web_bid: IWebBid; } +export interface IConfig extends ITimestamp { + id: number; + key_name: string; + value: string; + type: "string" | "number"; +} + export interface IPermission extends ITimestamp { id: number; name: string; diff --git a/auto-bid-server/bot-data/metadata.json b/auto-bid-server/bot-data/metadata.json index 224c96e..61870ed 100644 --- a/auto-bid-server/bot-data/metadata.json +++ b/auto-bid-server/bot-data/metadata.json @@ -1 +1 @@ -{"createdAt":1747701959077} \ No newline at end of file +{"createdAt":1747812172479} \ No newline at end of file diff --git a/auto-bid-server/package-lock.json b/auto-bid-server/package-lock.json index e45c632..14879ed 100644 --- a/auto-bid-server/package-lock.json +++ b/auto-bid-server/package-lock.json @@ -10,6 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@nestjs-modules/mailer": "^2.0.2", + "@nestjs/bull": "^11.0.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^4.0.1", "@nestjs/core": "^10.0.0", @@ -24,6 +25,7 @@ "@nestjs/websockets": "^11.0.11", "axios": "^1.8.3", "bcrypt": "^5.1.1", + "bull": "^4.16.5", "cheerio": "^1.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", @@ -31,6 +33,7 @@ "cookie-parser": "^1.4.7", "dayjs": "^1.11.13", "imap": "^0.8.19", + "ioredis": "^5.6.1", "lodash": "^4.17.21", "moment": "^2.30.1", "multer": "^1.4.5-lts.1", @@ -1649,6 +1652,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2337,6 +2346,84 @@ "license": "MIT", "peer": true }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nestjs-modules/mailer": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@nestjs-modules/mailer/-/mailer-2.0.2.tgz", @@ -2412,6 +2499,34 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/@nestjs/bull": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-11.0.2.tgz", + "integrity": "sha512-RjyP9JZUuLmMhmq1TMNIZqolkAd14az1jyXMMVki+C9dYvaMjWzBSwcZAtKs9Pk15Rm7qN1xn3R11aMV2Xv4gg==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^11.0.2", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", + "bull": "^3.3 || ^4.0.0" + } + }, + "node_modules/@nestjs/bull-shared": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.2.tgz", + "integrity": "sha512-dFlttJvBqIFD6M8JVFbkrR4Feb39OTAJPJpFVILU50NOJCM4qziRw3dSNG84Q3v+7/M6xUGMFdZRRGvBBKxoSA==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "10.4.9", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.4.9.tgz", @@ -4663,6 +4778,33 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/bull": { + "version": "4.16.5", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz", + "integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.9.0", + "get-port": "^5.1.1", + "ioredis": "^5.3.2", + "lodash": "^4.17.21", + "msgpackr": "^1.11.2", + "semver": "^7.5.2", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/bull/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -5068,6 +5210,15 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -5400,6 +5551,18 @@ "node": ">=18.x" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -7316,7 +7479,6 @@ "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", "license": "MIT", - "optional": true, "engines": { "node": ">=8" }, @@ -7904,6 +8066,30 @@ "node": ">=12.0.0" } }, + "node_modules/ioredis": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", + "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -9506,12 +9692,24 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", "license": "MIT" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -10421,6 +10619,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.4", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.4.tgz", + "integrity": "sha512-uaff7RG9VIC4jacFW9xzL3jc0iM32DNHe4jYVycBcjUePT/Klnfj7pqtWJt9khvDFizmjN2TlYniYmSS2LIaZg==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "1.4.5-lts.1", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz", @@ -10598,6 +10827,21 @@ } } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -11847,6 +12091,27 @@ "node": ">= 12.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -12979,6 +13244,12 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", diff --git a/auto-bid-server/package.json b/auto-bid-server/package.json index c1c33f1..f9f46af 100644 --- a/auto-bid-server/package.json +++ b/auto-bid-server/package.json @@ -26,6 +26,7 @@ }, "dependencies": { "@nestjs-modules/mailer": "^2.0.2", + "@nestjs/bull": "^11.0.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^4.0.1", "@nestjs/core": "^10.0.0", @@ -40,6 +41,7 @@ "@nestjs/websockets": "^11.0.11", "axios": "^1.8.3", "bcrypt": "^5.1.1", + "bull": "^4.16.5", "cheerio": "^1.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", @@ -47,6 +49,7 @@ "cookie-parser": "^1.4.7", "dayjs": "^1.11.13", "imap": "^0.8.19", + "ioredis": "^5.6.1", "lodash": "^4.17.21", "moment": "^2.30.1", "multer": "^1.4.5-lts.1", diff --git a/auto-bid-server/src/modules/app-configs/app-configs.module.ts b/auto-bid-server/src/modules/app-configs/app-configs.module.ts index c1df3a5..f13e642 100644 --- a/auto-bid-server/src/modules/app-configs/app-configs.module.ts +++ b/auto-bid-server/src/modules/app-configs/app-configs.module.ts @@ -1,8 +1,10 @@ -import { Module } from '@nestjs/common'; +import { BullModule } from '@nestjs/bull'; +import { Global, Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { EventEmitterModule } from '@nestjs/event-emitter'; import { ScheduleModule } from '@nestjs/schedule'; +@Global() @Module({ imports: [ ConfigModule.forRoot({ @@ -12,7 +14,17 @@ import { ScheduleModule } from '@nestjs/schedule'; wildcard: true, global: true, }), - ScheduleModule.forRoot() + BullModule.forRoot({ + redis: { + host: 'localhost', + port: 6379, + }, + }), + BullModule.registerQueue({ + name: 'mail-queue', + }), + ScheduleModule.forRoot(), ], + exports: [BullModule], }) export class AppConfigsModule {} diff --git a/auto-bid-server/src/modules/bids/bids.module.ts b/auto-bid-server/src/modules/bids/bids.module.ts index 2e873fc..4bc8d04 100644 --- a/auto-bid-server/src/modules/bids/bids.module.ts +++ b/auto-bid-server/src/modules/bids/bids.module.ts @@ -30,6 +30,7 @@ import { AdminDashboardController } from './controllers/admin/admin-dashboard.co import { TasksService } from './services/tasks.servise'; import { ConfigsService } from './services/configs.service'; import { Config } from './entities/configs.entity'; +import { AdminConfigsController } from './controllers/admin/admin-configs.controller'; @Module({ imports: [ @@ -55,6 +56,7 @@ import { Config } from './entities/configs.entity'; AdminWebBidsController, AdminSendMessageHistoriesController, AdminDashboardController, + AdminConfigsController, ], providers: [ BidsService, @@ -76,6 +78,7 @@ import { Config } from './entities/configs.entity'; SendMessageHistoriesService, BidsService, ConfigsService, + DashboardService, ], }) export class BidsModule {} diff --git a/auto-bid-server/src/modules/bids/controllers/admin/admin-configs.controller.ts b/auto-bid-server/src/modules/bids/controllers/admin/admin-configs.controller.ts new file mode 100644 index 0000000..71a7279 --- /dev/null +++ b/auto-bid-server/src/modules/bids/controllers/admin/admin-configs.controller.ts @@ -0,0 +1,20 @@ +import { Body, Controller, Get, Param, Post } from '@nestjs/common'; +import { DashboardService } from '../../services/dashboard.service'; +import { Config } from '../../entities/configs.entity'; +import { ConfigsService } from '../../services/configs.service'; +import { UpsertConfigDto } from '../../dto/config/upsert-config.dto'; + +@Controller('admin/configs') +export class AdminConfigsController { + constructor(private readonly configsService: ConfigsService) {} + + @Post('upsert') + async upsertConfig(@Body() data: UpsertConfigDto) { + return await this.configsService.upsertConfig(data); + } + + @Get(':key') + async getConfig(@Param('key') key: Config['key_name']) { + return await this.configsService.getConfigRes(key); + } +} diff --git a/auto-bid-server/src/modules/bids/dto/config/upsert-config.dto.ts b/auto-bid-server/src/modules/bids/dto/config/upsert-config.dto.ts new file mode 100644 index 0000000..1130b57 --- /dev/null +++ b/auto-bid-server/src/modules/bids/dto/config/upsert-config.dto.ts @@ -0,0 +1,13 @@ +import { IsEnum, IsNumber, IsOptional, IsString } from 'class-validator'; + +export class UpsertConfigDto { + @IsString() + key_name: string; + + @IsString() + @IsOptional() + value: string; + + @IsEnum(['string', 'number']) + type: 'string' | 'number'; +} diff --git a/auto-bid-server/src/modules/bids/services/bid-histories.service.ts b/auto-bid-server/src/modules/bids/services/bid-histories.service.ts index a38279b..6b01062 100644 --- a/auto-bid-server/src/modules/bids/services/bid-histories.service.ts +++ b/auto-bid-server/src/modules/bids/services/bid-histories.service.ts @@ -16,6 +16,8 @@ import { SendMessageHistoriesService } from './send-message-histories.service'; import { NotificationService } from '@/modules/notification/notification.service'; import { isTimeReached } from '@/ultils'; import { BidsService } from './bids.service'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { Event } from '../utils/events'; @Injectable() export class BidHistoriesService { @@ -28,6 +30,7 @@ export class BidHistoriesService { readonly sendMessageHistoriesService: SendMessageHistoriesService, private readonly notificationService: NotificationService, private readonly bidsService: BidsService, + private eventEmitter: EventEmitter2, ) {} async index() { @@ -38,6 +41,7 @@ export class BidHistoriesService { // Tìm thông tin bid từ database const bid = await this.bidsService.bidsRepo.findOne({ where: { id: bid_id }, + relations: { web_bid: true }, }); // Nếu không tìm thấy bid, trả về lỗi 404 @@ -104,6 +108,9 @@ export class BidHistoriesService { const botData = { ...bid, histories: response }; this.botTelegramApi.sendBidInfo(botData); + // Send event thống place bid + this.eventEmitter.emit(Event.BID_SUBMITED, botData); + // Lưu message đã gửi để theo dõi this.sendMessageHistoriesService.sendMessageRepo.save({ message: this.botTelegramApi.formatBidMessage(botData), diff --git a/auto-bid-server/src/modules/bids/services/bids.service.ts b/auto-bid-server/src/modules/bids/services/bids.service.ts index 2a809bd..fc371bd 100644 --- a/auto-bid-server/src/modules/bids/services/bids.service.ts +++ b/auto-bid-server/src/modules/bids/services/bids.service.ts @@ -153,7 +153,10 @@ export class BidsService { } async toggle(id: Bid['id']) { - const bid = await this.bidsRepo.findOne({ where: { id } }); + const bid = await this.bidsRepo.findOne({ + where: { id }, + relations: { web_bid: true }, + }); if (!bid) { throw new NotFoundException( @@ -301,7 +304,10 @@ export class BidsService { async outBid(id: Bid['id']) { const result = await this.bidsRepo.update(id, { status: 'out-bid' }); - const bid = await this.bidsRepo.findOne({ where: { id } }); + const bid = await this.bidsRepo.findOne({ + where: { id }, + relations: { web_bid: true }, + }); if (!result) throw new BadRequestException(AppResponse.toResponse(false)); @@ -353,7 +359,10 @@ export class BidsService { } async updateStatusByPrice(id: Bid['id'], data: UpdateStatusByPriceDto) { - const bid = await this.bidsRepo.findOne({ where: { id } }); + const bid = await this.bidsRepo.findOne({ + where: { id }, + relations: { web_bid: true }, + }); if (!bid) throw new NotFoundException( diff --git a/auto-bid-server/src/modules/bids/services/configs.service.ts b/auto-bid-server/src/modules/bids/services/configs.service.ts index 6823865..5a6c2d4 100644 --- a/auto-bid-server/src/modules/bids/services/configs.service.ts +++ b/auto-bid-server/src/modules/bids/services/configs.service.ts @@ -1,8 +1,14 @@ -import { Injectable } from '@nestjs/common'; +import { + BadRequestException, + HttpStatus, + Injectable, + NotFoundException, +} from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import { Config } from '../entities/configs.entity'; +import AppResponse from '@/response/app-response'; @Injectable() export class ConfigsService { @@ -29,4 +35,41 @@ export class ConfigsService { 'key_name', ]); } + + async getConfigRes(key_name: string) { + const result = await this.getConfig( + key_name as keyof typeof ConfigsService.CONFIG_KEYS, + ); + + if (!result) + throw new NotFoundException( + AppResponse.toResponse(null, { + message: 'Config key name not found', + status_code: HttpStatus.NOT_FOUND, + }), + ); + + return AppResponse.toResponse(result); + } + + async upsertConfig(data: Partial) { + let response = null; + + const prevConfig = await this.configRepo.findOne({ + where: { key_name: data.key_name }, + }); + + if (!prevConfig) { + response = await this.configRepo.save(data); + } else { + response = await this.configRepo.update( + { key_name: data.key_name }, + data, + ); + } + + if (!response) throw new BadRequestException(AppResponse.toResponse(false)); + + return AppResponse.toResponse(!!response); + } } diff --git a/auto-bid-server/src/modules/bids/utils/events.ts b/auto-bid-server/src/modules/bids/utils/events.ts index f3d7f3b..c2873fe 100644 --- a/auto-bid-server/src/modules/bids/utils/events.ts +++ b/auto-bid-server/src/modules/bids/utils/events.ts @@ -7,6 +7,8 @@ export class Event { public static ADMIN_BIDS_UPDATED = 'adminBidsUpdated'; public static WEB_UPDATED = 'webUpdated'; public static LOGIN_STATUS = 'login-status'; + public static BID_SUBMITED = 'bid-submited'; + public static BID_STATUS = 'bid-status'; public static verifyCode(data: WebBid) { return `${this.VERIFY_CODE}.${data.origin_url}`; diff --git a/auto-bid-server/src/modules/mails/mails.module.ts b/auto-bid-server/src/modules/mails/mails.module.ts index 468a3c1..9cc7c41 100644 --- a/auto-bid-server/src/modules/mails/mails.module.ts +++ b/auto-bid-server/src/modules/mails/mails.module.ts @@ -2,7 +2,8 @@ import { MailerModule } from '@nestjs-modules/mailer'; import { Module } from '@nestjs/common'; import { MailsService } from './services/mails.service'; import { ConfigModule, ConfigService } from '@nestjs/config'; - +import { BullModule } from '@nestjs/bull'; +import { MailProcessor } from './process/mail.processor'; @Module({ imports: [ MailerModule.forRootAsync({ @@ -21,7 +22,7 @@ import { ConfigModule, ConfigService } from '@nestjs/config'; inject: [ConfigService], }), ], - providers: [MailsService], + providers: [MailsService, MailProcessor], exports: [MailsService], }) export class MailsModule {} diff --git a/auto-bid-server/src/modules/mails/process/mail.processor.ts b/auto-bid-server/src/modules/mails/process/mail.processor.ts new file mode 100644 index 0000000..e0af756 --- /dev/null +++ b/auto-bid-server/src/modules/mails/process/mail.processor.ts @@ -0,0 +1,16 @@ +// processors/mail.processor.ts +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { MailsService } from '../services/mails.service'; + +@Processor('mail-queue') +export class MailProcessor { + constructor(private readonly mailsService: MailsService) {} + + @Process('send-mail') + async handleSendMail(job: Job) { + const { to, subject, html } = job.data; + + await this.mailsService.sendPlainHtml(to, subject, html); + } +} diff --git a/auto-bid-server/src/modules/mails/services/mails.service.ts b/auto-bid-server/src/modules/mails/services/mails.service.ts index d22bad5..a8600aa 100644 --- a/auto-bid-server/src/modules/mails/services/mails.service.ts +++ b/auto-bid-server/src/modules/mails/services/mails.service.ts @@ -1,11 +1,22 @@ import { Injectable } from '@nestjs/common'; import { MailerService } from '@nestjs-modules/mailer'; import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; -import { extractDomain } from '@/ultils'; +import { + extractDomain, + extractDomainSmart, + formatEndTime, + isTimeReached, +} from '@/ultils'; +import { Bid } from '@/modules/bids/entities/bid.entity'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; @Injectable() export class MailsService { - constructor(private readonly mailerService: MailerService) {} + constructor( + private readonly mailerService: MailerService, + @InjectQueue('mail-queue') private mailQueue: Queue, + ) {} async sendPlainText(to: string, subject: string, content: string) { await this.mailerService.sendMail({ @@ -15,6 +26,14 @@ export class MailsService { }); } + async sendHtmlMailJob(mailData: { + to: string; + subject: string; + html: string; + }) { + await this.mailQueue.add('send-mail', mailData); + } + async sendPlainHtml(to: string, subject: string, html: string) { const emails = to .split(',') @@ -33,17 +52,34 @@ export class MailsService { } generateProductTableHTML(products: ScrapItem[]): string { + if (!products.length) { + return ` + + + + + + Products + + +

Product Listing

+

No matching products found for your keywords today.

+ + + `; + } + const rows = products .map( (p) => ` - - Product Image - ${p.name} - $${p.current_price} - View - ${extractDomain(p.scrap_config.web_bid.origin_url)} - - `, + + Product Image + ${p.name} + ${p.current_price ? '$' + p.current_price : 'None'} + View + ${extractDomainSmart(p.scrap_config.web_bid.origin_url)} + + `, ) .join(''); @@ -78,12 +114,197 @@ export class MailsService { `; } - async sendWithTemplate(to: string, subject: string, payload: any) { - await this.mailerService.sendMail({ - to, - subject, - template: './welcome', // đường dẫn tương đối trong /templates - context: payload, // dữ liệu cho template - }); + getAuctionStatusEmailContent(bid: Bid): string { + const webname = extractDomain(bid.web_bid.origin_url); + const title = `[${webname}] ${bid.name || 'Unnamed Item'}`; + const endTime = formatEndTime(bid.close_time, false); + const competitor = `$${bid.current_price}`; + const max = `$${bid.max_price}`; + const submitted = `$${bid.max_price}`; + const nextBid = bid.max_price + bid.plus_price; + + const cardStyle = ` + max-width: 600px; + margin: 20px auto; + font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; + background: #ffffff; + border-radius: 8px; + box-shadow: 0 4px 8px rgba(0,0,0,0.1); + color: #333; + padding: 20px; + `; + + const headerStyle = (color: string) => + `font-size: 22px; font-weight: 700; color: ${color}; margin-bottom: 15px;`; + + const labelStyle = `font-weight: 600; width: 120px; display: inline-block; color: #555;`; + const valueStyle = `color: #222;`; + + const renderRow = (label: string, value: string) => + `

${label}: ${value}

`; + + switch (bid.status) { + case 'biding': + return ` +
+

✅ Auto Bid Started

+ ${renderRow('Title', title)} + ${renderRow('Max', max)} + ${renderRow('End time', endTime)} + ${renderRow('Competitor', competitor)} + ${renderRow('Bid submitted', submitted)} +
+ `; + + case 'out-bid': { + const overLimit = bid.current_price >= nextBid; + const belowReserve = bid.reserve_price > nextBid; + const timeExtended = bid.close_time ? 'Time extended' : 'No extension'; + + if (isTimeReached(bid.close_time)) { + return ` +
+

⏳ Auction Ended

+ ${renderRow('Title', title)} + ${renderRow('End time', endTime)} + ${renderRow('Final price', competitor)} +
+ `; + } + + if (overLimit || belowReserve) { + return ` +
+

⚠️ Outbid (${timeExtended})

+ ${renderRow('Title', title)} + ${renderRow('Competitor', competitor)} + ${renderRow('Max', max)} + ${renderRow('Next bid at', `$${nextBid}`)} + ${renderRow('End time', endTime)} +

⚠️ Current bid exceeds your max bid.

+
+ `; + } + + return ` +
+

🛑 Auction Canceled (${timeExtended})

+ ${renderRow('Title', title)} + ${renderRow('Competitor', competitor)} + ${renderRow('Max', max)} + ${renderRow('Next bid at', `$${nextBid}`)} + ${renderRow('End time', endTime)} +

🛑 Auction has been canceled.

+
+ `; + } + + case 'win-bid': + return ` +
+

🎉 You Won!

+ ${renderRow('Title', title)} + ${renderRow('Price won', `$${bid.current_price}`)} + ${renderRow('Max', max)} +
+ `; + + default: + return ` +
+

❓ Unknown Status

+ ${renderRow('Title', title)} +
+ `; + } + } + + getBidSubmittedEmailContent(bid: Bid): string { + const webname = extractDomain(bid.web_bid.origin_url); + const title = `[${webname}] ${bid.name || 'Unnamed Item'}`; + const endTime = formatEndTime(bid.close_time, false); + const competitor = `$${bid.current_price}`; + const max = `$${bid.max_price}`; + const submitted = `$${bid.max_price}`; + const maxReached = bid.max_price <= bid.max_price; + + return ` + + + + + + +
+

Bid Submitted${bid.close_time ? ', Time extended' : ', No extension'}${maxReached ? ' * MAX REACH *' : ''}

+ + + + + + + + + + + + + + + + + + + + + + + +
Title${title}
Competitor${competitor}
Bid Submitted${submitted} ${maxReached ? '(***MAXIMUM REACH***)' : ''}
Max${max}
End Time${endTime}
+
+ + + `; } } diff --git a/auto-bid-server/src/modules/notification/constants/index.ts b/auto-bid-server/src/modules/notification/constants/index.ts index 19a4b18..ca1d7fe 100644 --- a/auto-bid-server/src/modules/notification/constants/index.ts +++ b/auto-bid-server/src/modules/notification/constants/index.ts @@ -1,3 +1,3 @@ -export const NAME_EVENTS = { - BID_STATUS: 'notify.bid-status', -}; +// export const NAME_EVENTS = { +// BID_STATUS: 'notify.bid-status', +// }; diff --git a/auto-bid-server/src/modules/notification/controllers/client-notification.controller.ts b/auto-bid-server/src/modules/notification/controllers/client-notification.controller.ts index 93e46fa..f68735d 100644 --- a/auto-bid-server/src/modules/notification/controllers/client-notification.controller.ts +++ b/auto-bid-server/src/modules/notification/controllers/client-notification.controller.ts @@ -20,7 +20,8 @@ export class ClientNotificationController { @Post('test') async test() { const bid = await this.bidsService.bidsRepo.findOne({ - where: { lot_id: '26077023' }, + where: { lot_id: '23755862' }, + relations: { web_bid: true }, }); return await this.notifyService.emitBidStatus({ diff --git a/auto-bid-server/src/modules/notification/listeners/admin-notification.listener.ts b/auto-bid-server/src/modules/notification/listeners/admin-notification.listener.ts index 5b4e007..7200c84 100644 --- a/auto-bid-server/src/modules/notification/listeners/admin-notification.listener.ts +++ b/auto-bid-server/src/modules/notification/listeners/admin-notification.listener.ts @@ -1,16 +1,23 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; -import { NAME_EVENTS } from '../constants'; import { Bid } from '@/modules/bids/entities/bid.entity'; import { Notification } from '../entities/notification.entity'; import { BotTelegramApi } from '@/modules/bids/apis/bot-telegram.api'; +import { MailsService } from '@/modules/mails/services/mails.service'; +import { ConfigsService } from '@/modules/bids/services/configs.service'; +import * as moment from 'moment'; +import { Event } from '@/modules/bids/utils/events'; @Injectable() export class AdminNotificationListener { - constructor(private readonly botTelegramApi: BotTelegramApi) {} + constructor( + private readonly botTelegramApi: BotTelegramApi, + private readonly mailsService: MailsService, + private readonly configsSerice: ConfigsService, + ) {} - @OnEvent(NAME_EVENTS.BID_STATUS) - handleBidStatus({ + @OnEvent(Event.BID_STATUS) + async handleBidStatus({ bid, notification, }: { @@ -20,5 +27,30 @@ export class AdminNotificationListener { if (JSON.parse(notification.send_to).length <= 0) return; this.botTelegramApi.sendMessage(notification.message); + + const mails = + (await this.configsSerice.getConfig('MAIL_SCRAP_REPORT')).value || ''; + + this.mailsService.sendHtmlMailJob({ + to: mails, + html: this.mailsService.getAuctionStatusEmailContent(bid), + subject: + 'Report Auto Auctions System ' + + moment(new Date()).format('YYYY-MM-DD HH:mm'), + }); + } + + @OnEvent(Event.BID_SUBMITED) + async handleBidSubmited(bid: Bid) { + const mails = + (await this.configsSerice.getConfig('MAIL_SCRAP_REPORT')).value || ''; + + this.mailsService.sendHtmlMailJob({ + to: mails, + html: this.mailsService.getBidSubmittedEmailContent(bid), + subject: + 'Report Auto Auctions System ' + + moment(new Date()).format('YYYY-MM-DD HH:mm'), + }); } } diff --git a/auto-bid-server/src/modules/notification/notification.module.ts b/auto-bid-server/src/modules/notification/notification.module.ts index b01ffe3..49d5c8a 100644 --- a/auto-bid-server/src/modules/notification/notification.module.ts +++ b/auto-bid-server/src/modules/notification/notification.module.ts @@ -8,11 +8,13 @@ import { AdminNotificationListener } from './listeners/admin-notification.listen import { NotificationService } from './notification.service'; import { SendMessageHistoriesService } from '../bids/services/send-message-histories.service'; import { SendMessageHistory } from '../bids/entities/send-message-histories.entity'; +import { MailsModule } from '../mails/mails.module'; @Module({ imports: [ forwardRef(() => BidsModule), TypeOrmModule.forFeature([Notification, SendMessageHistory]), + MailsModule, ], controllers: [NotificationController, ClientNotificationController], providers: [NotificationService, AdminNotificationListener], diff --git a/auto-bid-server/src/modules/notification/notification.service.ts b/auto-bid-server/src/modules/notification/notification.service.ts index f6e2c01..4592f00 100644 --- a/auto-bid-server/src/modules/notification/notification.service.ts +++ b/auto-bid-server/src/modules/notification/notification.service.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { Bid } from '../bids/entities/bid.entity'; -import { NAME_EVENTS } from './constants'; import { BotTelegramApi } from '../bids/apis/bot-telegram.api'; import { InjectRepository } from '@nestjs/typeorm'; import { Notification } from './entities/notification.entity'; @@ -17,6 +16,7 @@ import { Column } from 'nestjs-paginate/lib/helper'; import AppResponse from '@/response/app-response'; import { SendMessageHistoriesService } from '../bids/services/send-message-histories.service'; import { SendMessageHistory } from '../bids/entities/send-message-histories.entity'; +import { Event } from '../bids/utils/events'; @Injectable() export class NotificationService { @@ -116,7 +116,7 @@ export class NotificationService { message: notification.message, type: bid.status, max_price: bid.max_price, - reserve_price: bid.reserve_price + reserve_price: bid.reserve_price, }, }); @@ -127,13 +127,13 @@ export class NotificationService { message: notification.message, type: bid.status, max_price: bid.max_price, - reserve_price: bid.reserve_price + reserve_price: bid.reserve_price, }); - this.eventEmitter.emit(NAME_EVENTS.BID_STATUS, { + this.eventEmitter.emit(Event.BID_STATUS, { bid: { ...bid, - status: 'out-bid', + // status: 'out-bid', }, notification, }); diff --git a/auto-bid-server/src/modules/scraps/controllers/client/scrap-configs.controller.ts b/auto-bid-server/src/modules/scraps/controllers/client/scrap-configs.controller.ts new file mode 100644 index 0000000..7f2c848 --- /dev/null +++ b/auto-bid-server/src/modules/scraps/controllers/client/scrap-configs.controller.ts @@ -0,0 +1,12 @@ +import { Controller, Get } from '@nestjs/common'; +import { ScrapConfigsService } from '../../services/scrap-config.service'; + +@Controller('scrap-configs') +export class ClientScrapConfigsController { + constructor(private readonly scrapConfigsService: ScrapConfigsService) {} + + @Get() + async clientGetScrapeConfigs() { + return await this.scrapConfigsService.clientGetScrapeConfigs(); + } +} diff --git a/auto-bid-server/src/modules/scraps/controllers/client/scrap-items.controller.ts b/auto-bid-server/src/modules/scraps/controllers/client/scrap-items.controller.ts new file mode 100644 index 0000000..edec506 --- /dev/null +++ b/auto-bid-server/src/modules/scraps/controllers/client/scrap-items.controller.ts @@ -0,0 +1,17 @@ +import { Body, Controller, Get, Post } from '@nestjs/common'; +import { ScrapConfigsService } from '../../services/scrap-config.service'; +import { ScrapItemsService } from '../../services/scrap-item-config.service'; +import { UpsertScrapItemDto } from '../../dto/scrap-items/upsert-scrap-item.dto'; +import { ScrapItem } from '../../entities/scrap-item.entity'; + +@Controller('scrap-items') +export class ClientScrapItemsController { + constructor(private readonly scrapItemsService: ScrapItemsService) {} + + @Post('upsert') + async upsertScrapItems(@Body() data: UpsertScrapItemDto[]) { + return await this.scrapItemsService.upsertScrapItemsRes( + data as ScrapItem[], + ); + } +} diff --git a/auto-bid-server/src/modules/scraps/controllers/scrap-config.controller.ts b/auto-bid-server/src/modules/scraps/controllers/scrap-config.controller.ts index ed9be5d..6d348da 100644 --- a/auto-bid-server/src/modules/scraps/controllers/scrap-config.controller.ts +++ b/auto-bid-server/src/modules/scraps/controllers/scrap-config.controller.ts @@ -1,22 +1,12 @@ -import { MailsService } from '@/modules/mails/services/mails.service'; -import { Body, Controller, Get, Param, Post, Put } from '@nestjs/common'; -import { Between, IsNull, Not } from 'typeorm'; +import { Body, Controller, Param, Post, Put } from '@nestjs/common'; import { CreateScrapConfigDto } from '../dto/scrap-config/create-scrap-config'; import { UpdateScrapConfigDto } from '../dto/scrap-config/update-scrap-config'; import { ScrapConfig } from '../entities/scrap-config.entity'; import { ScrapConfigsService } from '../services/scrap-config.service'; -import { ScrapItemsService } from '../services/scrap-item-config.service'; -import { ConfigsService } from '@/modules/bids/services/configs.service'; -import * as moment from 'moment'; @Controller('admin/scrap-configs') export class ScrapConfigsController { - constructor( - private readonly scrapConfigsService: ScrapConfigsService, - private readonly scrapItemsService: ScrapItemsService, - private readonly mailsService: MailsService, - private readonly configsSerivce: ConfigsService, - ) {} + constructor(private readonly scrapConfigsService: ScrapConfigsService) {} @Post() async create(@Body() data: CreateScrapConfigDto) { @@ -30,56 +20,4 @@ export class ScrapConfigsController { ) { return await this.scrapConfigsService.update(id, data); } - - @Get() - async test() { - const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT')) - .value; - - if (!mails) return; - - const scrapConfigs = await this.scrapConfigsService.scrapConfigRepo.find({ - where: { - search_url: Not(IsNull()), - keywords: Not(IsNull()), - }, - relations: { - web_bid: true, - }, - }); - const models = this.scrapConfigsService.scrapModels(scrapConfigs); - await Promise.allSettled( - models.map(async (item) => { - await item.action(); - for (const key of Object.keys(item.results)) { - const dataArray = item.results[key]; - const result = - await this.scrapItemsService.upsertScrapItems(dataArray); - console.log(result); - } - }), - ); - - const startOfDay = new Date(); - startOfDay.setHours(0, 0, 0, 0); - - const endOfDay = new Date(); - endOfDay.setHours(23, 59, 59, 999); - - const data = await this.scrapItemsService.scrapItemRepo.find({ - where: { - updated_at: Between(startOfDay, endOfDay), - }, - relations: { scrap_config: { web_bid: true } }, - order: { updated_at: 'DESC' }, - }); - - await this.mailsService.sendPlainHtml( - mails, - `Auction Items Matching Your Keywords – Daily Update ${moment(new Date()).format('YYYY-MM-DD HH:mm')}`, - this.mailsService.generateProductTableHTML(data), - ); - - return { a: 'abc' }; - } } diff --git a/auto-bid-server/src/modules/scraps/dto/scrap-items/upsert-scrap-item.dto.ts b/auto-bid-server/src/modules/scraps/dto/scrap-items/upsert-scrap-item.dto.ts new file mode 100644 index 0000000..2a9c902 --- /dev/null +++ b/auto-bid-server/src/modules/scraps/dto/scrap-items/upsert-scrap-item.dto.ts @@ -0,0 +1,18 @@ +import { IsNumber, IsString, IsUrl } from 'class-validator'; + +export class UpsertScrapItemDto { + @IsUrl() + url: string; + @IsString() + image_url: string; + @IsString() + name: string; + @IsString() + keyword: string; + @IsNumber() + current_price: number; + @IsNumber() + scrap_config_id: number; + @IsString() + model: string; +} diff --git a/auto-bid-server/src/modules/scraps/models/scrap-interface.ts b/auto-bid-server/src/modules/scraps/models/scrap-interface.ts deleted file mode 100644 index 16d8a8d..0000000 --- a/auto-bid-server/src/modules/scraps/models/scrap-interface.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ScrapItem } from '../entities/scrap-item.entity'; - -export interface ScrapInterface { - getItemsInHtml: (data: { - html: string; - keyword: string; - }) => Promise; - - action: () => Promise; -} diff --git a/auto-bid-server/src/modules/scraps/models/scrap-model.ts b/auto-bid-server/src/modules/scraps/models/scrap-model.ts deleted file mode 100644 index 4129392..0000000 --- a/auto-bid-server/src/modules/scraps/models/scrap-model.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { ScrapConfig } from '../entities/scrap-config.entity'; -import { ScrapItem } from '../entities/scrap-item.entity'; -import { ScrapInterface } from './scrap-interface'; -import { Element } from 'domhandler'; -export class ScrapModel implements ScrapInterface { - protected keywords: string; - protected search_url: string; - protected scrap_config_id: ScrapConfig['id']; - public results: Record = {}; - - constructor({ - keywords, - search_url, - scrap_config_id, - }: { - keywords: string; - search_url: string; - scrap_config_id: ScrapConfig['id']; - }) { - this.keywords = keywords; - this.search_url = search_url; - this.scrap_config_id = scrap_config_id; - } - - protected buildUrlWithKey(rawUrl: string, keyword: string) { - return rawUrl.replaceAll('{{keyword}}', keyword); - } - - protected extractUrls(): { url: string; keyword: string }[] { - // Tách từng key ra từ một chuỗi keywords - const keywordList = this.keywords.split(', '); - - // Dừng hàm nếu không có key nào - if (keywordList.length <= 0) return []; - - // Lập qua từng key để lấy url - return keywordList.map((keyword) => { - return { - url: this.buildUrlWithKey(this.search_url, keyword), - keyword, - }; - }); - } - - action: () => Promise; - - getInfoItems: ( - data: { name: string; el: Element }[], - ) => Record[]; - - getItemsInHtml: (data: { - html: string; - keyword: string; - }) => Promise; - - protected filterItemByKeyword = (keyword: string, data: ScrapItem[]) => { - return data.filter((item) => - item.name.toLowerCase().includes(keyword.toLowerCase()), - ); - }; -} diff --git a/auto-bid-server/src/modules/scraps/models/www.grays.com/grays-scrap-model.ts b/auto-bid-server/src/modules/scraps/models/www.grays.com/grays-scrap-model.ts deleted file mode 100644 index ebbe485..0000000 --- a/auto-bid-server/src/modules/scraps/models/www.grays.com/grays-scrap-model.ts +++ /dev/null @@ -1,83 +0,0 @@ -import axios from 'axios'; -import { ScrapInterface } from '../scrap-interface'; -import { ScrapModel } from '../scrap-model'; -import * as cheerio from 'cheerio'; -import { Element } from 'domhandler'; -import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; -import { extractModelId, extractNumber } from '@/ultils'; -export class GraysScrapModel extends ScrapModel { - action = async () => { - const urls = this.extractUrls(); - - const results = await Promise.allSettled( - urls.map(async (item) => ({ - html: (await axios.get(item.url)).data, - keyword: item.keyword, - })), - ); - - const htmlsData = results - .filter((res) => res.status === 'fulfilled') - .map((res) => (res as PromiseFulfilledResult).value); - - await Promise.all( - htmlsData.map(async (cur) => { - try { - const data = await this.getItemsInHtml(cur); - const results = this.filterItemByKeyword(cur.keyword, data); - - this.results[cur.keyword] = results; // hoặc push như gợi ý trên - return results; - } catch (err) { - console.error(`❌ Error with keyword ${cur.keyword}:`, err); - return []; // fallback - } - }), - ); - }; - - getPriceByEl = ($: cheerio.CheerioAPI, el: Element): number | null => { - const selectors = [ - '.sc-ijDOKB.sc-bStcSt.ikmQUw.eEycyP', // Single product price - '.sc-ijDOKB.ikmQUw', // Multiple product price - ]; - - for (const selector of selectors) { - const text = $(el).find(selector).text(); - const price = extractNumber(text); - if (price) return price; - } - - return null; - }; - - getItemsInHtml = async ({ - html, - keyword, - }: { - html: string; - keyword: string; - }) => { - const $ = cheerio.load(html); - - const container = $('.sc-102aeaf3-1.eYPitT'); - - const items = container.children('div').toArray(); - - const results = items.map((el) => { - const url = $(el).find('.sc-pKqro.sc-gFnajm.gqkMpZ.dzWUkJ').attr('href'); - - return { - name: $(el).find('.sc-jlGgGc.dJRywx').text().trim(), - image_url: $(el).find('img.sc-gtJxfw.jbgdlx').attr('src'), - model: extractModelId(url), - keyword, - url, - current_price: this.getPriceByEl($, el), - scrap_config_id: this.scrap_config_id, - } as ScrapItem; - }); - - return results; - }; -} diff --git a/auto-bid-server/src/modules/scraps/models/www.langtons.com.au/langtons-scrap-model.ts b/auto-bid-server/src/modules/scraps/models/www.langtons.com.au/langtons-scrap-model.ts deleted file mode 100644 index 3b24ada..0000000 --- a/auto-bid-server/src/modules/scraps/models/www.langtons.com.au/langtons-scrap-model.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { ScrapItem } from '@/modules/scraps/entities/scrap-item.entity'; -import { extractModelId, extractNumber } from '@/ultils'; -import axios from 'axios'; -import * as cheerio from 'cheerio'; -import { Element } from 'domhandler'; -import { ScrapModel } from '../scrap-model'; - -export class LangtonsScrapModel extends ScrapModel { - action = async () => { - const urls = this.extractUrls(); - - console.log({ urls }); - const results = await Promise.allSettled( - urls.map(async (item) => ({ - html: ( - await axios.get(item.url, { - headers: { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', - Accept: 'text/html', - }, - maxRedirects: 5, // default là 5, bạn có thể tăng lên - }) - ).data, - keyword: item.keyword, - })), - ); - - console.log( - results.map((r) => ({ - status: r.status, - reason: - r.status === 'rejected' - ? (r as PromiseRejectedResult).reason.message - : undefined, - })), - ); - - const htmlsData = results - .filter((res) => res.status === 'fulfilled') - .map((res) => (res as PromiseFulfilledResult).value); - - console.log({ htmlsData }); - - await Promise.all( - htmlsData.map(async (cur) => { - try { - const data = await this.getItemsInHtml(cur); - const results = this.filterItemByKeyword(cur.keyword, data); - - this.results[cur.keyword] = results; // hoặc push như gợi ý trên - return results; - } catch (err) { - console.error(`❌ Error with keyword ${cur.keyword}:`, err); - return []; // fallback - } - }), - ); - }; - - getPriceByEl = ($: cheerio.CheerioAPI, el: Element): number | null => { - const selectors = [ - '.sc-ijDOKB.sc-bStcSt.ikmQUw.eEycyP', // Single product price - '.sc-ijDOKB.ikmQUw', // Multiple product price - ]; - - for (const selector of selectors) { - const text = $(el).find(selector).text(); - const price = extractNumber(text); - if (price) return price; - } - - return null; - }; - - getItemsInHtml = async ({ - html, - keyword, - }: { - html: string; - keyword: string; - }) => { - const $ = cheerio.load(html); - - const container = $('.row.product-grid.grid-view'); - - const items = container.children('div').toArray(); - - const results = items.map((el) => { - const url = $(el).find('a.js-pdp-link.pdp-link-anchor').attr('href'); - - return { - name: $(el).find('.link.js-pdp-link').text().trim(), - image_url: $(el).find('img.tile-image.loaded').attr('src'), - model: extractModelId(url), - keyword, - url, - current_price: extractNumber( - $(el).find('div.max-bid-price.price').text(), - ), - scrap_config_id: this.scrap_config_id, - } as ScrapItem; - }); - - console.log({ results }); - return results; - }; -} diff --git a/auto-bid-server/src/modules/scraps/scraps.module.ts b/auto-bid-server/src/modules/scraps/scraps.module.ts index ebf56cc..a0f4c1f 100644 --- a/auto-bid-server/src/modules/scraps/scraps.module.ts +++ b/auto-bid-server/src/modules/scraps/scraps.module.ts @@ -8,6 +8,8 @@ import { TasksService } from './services/tasks.service'; import { ScrapItemsService } from './services/scrap-item-config.service'; import { MailsModule } from '../mails/mails.module'; import { BidsModule } from '../bids/bids.module'; +import { ClientScrapConfigsController } from './controllers/client/scrap-configs.controller'; +import { ClientScrapItemsController } from './controllers/client/scrap-items.controller'; @Module({ imports: [ @@ -17,6 +19,10 @@ import { BidsModule } from '../bids/bids.module'; ], providers: [ScrapConfigsService, TasksService, ScrapItemsService], exports: [ScrapConfigsService, TasksService, ScrapItemsService], - controllers: [ScrapConfigsController], + controllers: [ + ScrapConfigsController, + ClientScrapConfigsController, + ClientScrapItemsController, + ], }) export class ScrapsModule {} diff --git a/auto-bid-server/src/modules/scraps/services/scrap-config.service.ts b/auto-bid-server/src/modules/scraps/services/scrap-config.service.ts index 3f8c58b..0918982 100644 --- a/auto-bid-server/src/modules/scraps/services/scrap-config.service.ts +++ b/auto-bid-server/src/modules/scraps/services/scrap-config.service.ts @@ -1,15 +1,11 @@ -import { Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; -import { ScrapConfig } from '../entities/scrap-config.entity'; import AppResponse from '@/response/app-response'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { plainToClass } from 'class-transformer'; +import { IsNull, Not, Repository } from 'typeorm'; import { CreateScrapConfigDto } from '../dto/scrap-config/create-scrap-config'; import { UpdateScrapConfigDto } from '../dto/scrap-config/update-scrap-config'; -import axios from 'axios'; -import { WebBid } from '@/modules/bids/entities/wed-bid.entity'; -import { GraysScrapModel } from '../models/www.grays.com/grays-scrap-model'; -import { LangtonsScrapModel } from '../models/www.langtons.com.au/langtons-scrap-model'; +import { ScrapConfig } from '../entities/scrap-config.entity'; @Injectable() export class ScrapConfigsService { @@ -18,6 +14,17 @@ export class ScrapConfigsService { readonly scrapConfigRepo: Repository, ) {} + async clientGetScrapeConfigs() { + const data = await this.scrapConfigRepo.find({ + where: { search_url: Not(IsNull()), keywords: Not(IsNull()) }, + relations: { + web_bid: true, + }, + }); + + return AppResponse.toResponse(plainToClass(ScrapConfig, data)); + } + async create(data: CreateScrapConfigDto) { const result = await this.scrapConfigRepo.save({ search_url: data.search_url, @@ -40,22 +47,4 @@ export class ScrapConfigsService { return AppResponse.toResponse(true); } - - scrapModel(scrapConfig: ScrapConfig) { - switch (scrapConfig.web_bid.origin_url) { - case 'https://www.grays.com': { - return new GraysScrapModel({ - ...scrapConfig, - scrap_config_id: scrapConfig.id, - }); - } - default: { - return null; - } - } - } - - scrapModels(data: ScrapConfig[]) { - return data.map((item) => this.scrapModel(item)).filter((item) => !!item); - } } diff --git a/auto-bid-server/src/modules/scraps/services/scrap-item-config.service.ts b/auto-bid-server/src/modules/scraps/services/scrap-item-config.service.ts index 76d7f4a..0ef6c70 100644 --- a/auto-bid-server/src/modules/scraps/services/scrap-item-config.service.ts +++ b/auto-bid-server/src/modules/scraps/services/scrap-item-config.service.ts @@ -1,7 +1,8 @@ -import { Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import { ScrapItem } from '../entities/scrap-item.entity'; +import AppResponse from '@/response/app-response'; @Injectable() export class ScrapItemsService { @@ -58,4 +59,12 @@ export class ScrapItemsService { updated: toUpdate.length, }; } + + async upsertScrapItemsRes(items: ScrapItem[]) { + const rs = await this.upsertScrapItems(items); + + if (!rs) throw new BadRequestException(AppResponse.toResponse(null)); + + return AppResponse.toResponse(rs); + } } diff --git a/auto-bid-server/src/modules/scraps/services/tasks.service.ts b/auto-bid-server/src/modules/scraps/services/tasks.service.ts index 4efc258..9b48f6c 100644 --- a/auto-bid-server/src/modules/scraps/services/tasks.service.ts +++ b/auto-bid-server/src/modules/scraps/services/tasks.service.ts @@ -1,11 +1,13 @@ +import { ConfigsService } from '@/modules/bids/services/configs.service'; +import { DashboardService } from '@/modules/bids/services/dashboard.service'; +import { MailsService } from '@/modules/mails/services/mails.service'; +import { delay } from '@/ultils'; import { Injectable, Logger } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; -import { Between, IsNull, Not } from 'typeorm'; +import { Cron } from '@nestjs/schedule'; +import * as moment from 'moment'; +import { Between } from 'typeorm'; import { ScrapConfigsService } from './scrap-config.service'; import { ScrapItemsService } from './scrap-item-config.service'; -import { MailsService } from '@/modules/mails/services/mails.service'; -import { ConfigsService } from '@/modules/bids/services/configs.service'; -import * as moment from 'moment'; @Injectable() export class TasksService { @@ -16,57 +18,79 @@ export class TasksService { private readonly scrapItemsService: ScrapItemsService, private readonly mailsService: MailsService, private readonly configsSerivce: ConfigsService, + private readonly dashboardService: DashboardService, ) {} - @Cron('0 2 * * *') - async handleScraps() { + async runProcessAndSendReport(processName: string) { const mails = (await this.configsSerivce.getConfig('MAIL_SCRAP_REPORT')) - .value; + ?.value; + if (!mails) { + console.warn('No mails configured for report. Skipping.'); + return; + } - if (!mails) return; + // Nếu process đang chạy, không chạy lại + const initialStatus = + await this.dashboardService.getStatusProcessByName(processName); + if (initialStatus === 'online') { + console.log( + `Process ${processName} is already running. Skipping execution.`, + ); + return; + } - const scrapConfigs = await this.scrapConfigsService.scrapConfigRepo.find({ - where: { - search_url: Not(IsNull()), - keywords: Not(IsNull()), - }, - relations: { - web_bid: true, - }, - }); - const models = this.scrapConfigsService.scrapModels(scrapConfigs); - await Promise.allSettled( - models.map(async (item) => { - await item.action(); - for (const key of Object.keys(item.results)) { - const dataArray = item.results[key]; - const result = - await this.scrapItemsService.upsertScrapItems(dataArray); - console.log(result); - } - }), - ); + // Reset và chạy process + await this.dashboardService.resetProcessByName(processName); + console.log(`Process ${processName} started.`); + // Đợi process kết thúc, có timeout + const maxAttempts = 60; // 10 phút + let attempts = 0; + let status = 'online'; + + while (status === 'online' && attempts < maxAttempts) { + await delay(10000); // 10 giây + status = await this.dashboardService.getStatusProcessByName(processName); + attempts++; + } + + if (status === 'online') { + console.warn( + `Process ${processName} still running after timeout. Skipping report.`, + ); + return; + } + + // Khi process kết thúc => gửi mail const startOfDay = new Date(); startOfDay.setHours(0, 0, 0, 0); - const endOfDay = new Date(); endOfDay.setHours(23, 59, 59, 999); - const data = await this.scrapItemsService.scrapItemRepo.find({ - where: { - updated_at: Between(startOfDay, endOfDay), - }, - relations: { scrap_config: { web_bid: true } }, - order: { updated_at: 'DESC' }, - }); + try { + const data = await this.scrapItemsService.scrapItemRepo.find({ + where: { + updated_at: Between(startOfDay, endOfDay), + }, + relations: { scrap_config: { web_bid: true } }, + order: { updated_at: 'DESC' }, + }); - await this.mailsService.sendPlainHtml( - mails, - `Auction Items Matching Your Keywords – Daily Update ${moment(new Date()).format('YYYY-MM-DD HH:mm')}`, - this.mailsService.generateProductTableHTML(data), - ); + await this.mailsService.sendHtmlMailJob({ + to: mails, + subject: `Auction Items Matching Your Keywords – Daily Update ${moment().format('YYYY-MM-DD HH:mm')}`, + html: this.mailsService.generateProductTableHTML(data), + }); - console.log('Send report success'); + console.log('Report mail sent successfully.'); + } catch (err) { + console.error('Failed to generate or send report:', err); + } + } + + @Cron('0 2 * * *') + async handleScraps() { + const processName = 'scrape-data-keyword'; + await this.runProcessAndSendReport(processName); } } diff --git a/auto-bid-server/src/ultils/index.ts b/auto-bid-server/src/ultils/index.ts index 1389ad3..7d9e5d0 100644 --- a/auto-bid-server/src/ultils/index.ts +++ b/auto-bid-server/src/ultils/index.ts @@ -1,4 +1,5 @@ import { Bid } from '@/modules/bids/entities/bid.entity'; +import * as moment from 'moment'; export function extractModelId(url: string): string | null { switch (extractDomain(url)) { @@ -31,6 +32,25 @@ export function subtractMinutes(timeStr: string, minutes: number) { return date.toISOString(); // Trả về dạng chuẩn ISO } +export function extractDomainSmart(url: string) { + const PUBLIC_SUFFIXES = ['com.au', 'co.uk', 'com.vn', 'org.au', 'gov.uk']; + + try { + const hostname = new URL(url).hostname.replace(/^www\./, ''); // remove "www." + const parts = hostname.split('.'); + + for (let i = 0; i < PUBLIC_SUFFIXES.length; i++) { + if (hostname.endsWith(PUBLIC_SUFFIXES[i])) { + return parts[parts.length - PUBLIC_SUFFIXES[i].split('.').length - 1]; + } + } + + return parts[parts.length - 2]; + } catch (e) { + return url; + } +} + export function subtractSeconds(time: string, seconds: number) { const date = new Date(time); date.setSeconds(date.getSeconds() - seconds); @@ -185,3 +205,13 @@ export function extractNumber(str: string) { const match = str.match(/\d+(\.\d+)?/); return match ? parseFloat(match[0]) : null; } + +export function formatEndTime( + closeTime: string | Date, + extended: boolean, +): string { + return `${moment(closeTime).format('YYYY-MM-DD HH:mm')} (${extended ? 'extended' : 'no extension'})`; +} + +export const delay = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/scrape-data-keyword/.gitignore b/scrape-data-keyword/.gitignore new file mode 100644 index 0000000..e0b5f67 --- /dev/null +++ b/scrape-data-keyword/.gitignore @@ -0,0 +1,57 @@ +# compiled output +/dist +/node_modules +/build +/public +/system/error-images +/system/profiles +/system/local-data + +# Logs +logs +*.log +npm-debug.log* +pnpm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# OS +.DS_Store + +# Tests +/coverage +/.nyc_output + +# IDEs and editors +/.idea +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace + +# IDE - VSCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +# dotenv environment variable files +.env +.env.local + +# temp directory +.temp +.tmp + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/scrape-data-keyword/apis/scrape.js b/scrape-data-keyword/apis/scrape.js new file mode 100644 index 0000000..d4f696e --- /dev/null +++ b/scrape-data-keyword/apis/scrape.js @@ -0,0 +1,38 @@ +import axios from "../system/axios.js"; + +export const getScapeConfigs = async () => { + try { + const { data } = await axios({ + method: "GET", + url: "scrap-configs", + }); + + if (!data || !data?.data) { + return []; + } + + return data.data; + } catch (error) { + console.log("❌ ERROR IN SERVER (getScapeConfigs): ", error); + return []; + } +}; + +export const upsertScapeItems = async (values) => { + try { + const { data } = await axios({ + method: "POST", + url: "scrap-items/upsert", + data: values, + }); + + if (!data || !data?.data) { + return []; + } + + return data.data; + } catch (error) { + console.log("❌ ERROR IN SERVER (upsertScapeItems): ", error); + return []; + } +}; diff --git a/scrape-data-keyword/ecosystem.config.cjs b/scrape-data-keyword/ecosystem.config.cjs new file mode 100644 index 0000000..ed58f0c --- /dev/null +++ b/scrape-data-keyword/ecosystem.config.cjs @@ -0,0 +1,17 @@ +module.exports = { + apps: [ + { + name: "scrape-data-keyword", + script: "./index.js", + instances: 1, + exec_mode: "fork", + watch: false, + log_date_format: "YYYY-MM-DD HH:mm:ss", + output: "./logs/out.log", + error: "./logs/error.log", + merge_logs: true, + max_memory_restart: "1G", + autorestart: false, + }, + ], +}; diff --git a/scrape-data-keyword/index.js b/scrape-data-keyword/index.js new file mode 100644 index 0000000..247c76d --- /dev/null +++ b/scrape-data-keyword/index.js @@ -0,0 +1,44 @@ +import dotenv from "dotenv"; +import "dotenv/config"; +dotenv.config(); +import { getScapeConfigs, upsertScapeItems } from "./apis/scrape.js"; +import { ScrapConfigsService } from "./services/scrap-configs-service.js"; +import browser from "./system/browser.js"; + +const init = async () => { + const scrapConfigs = await getScapeConfigs(); + + try { + const page = await browser.newPage(); + + await page.setUserAgent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" + ); + + const models = ScrapConfigsService.scrapModels(scrapConfigs, page); + console.log(`Loaded ${models.length} scrape models`); + + for (let model of models) { + try { + await model.action(); + + for (const key of Object.keys(model.results)) { + const dataArray = model.results[key]; + const result = await upsertScapeItems(dataArray); + console.log(`Upserted ${dataArray.length} items for ${key}:`, result); + } + } catch (err) { + console.error( + `Error in model ${model.config?.name || "unknown"}:`, + err + ); + } + } + + await page.close(); + } finally { + await browser.close(); + } +}; + +init(); diff --git a/scrape-data-keyword/models/allbids-scrap-model.js b/scrape-data-keyword/models/allbids-scrap-model.js new file mode 100644 index 0000000..9c8a2f7 --- /dev/null +++ b/scrape-data-keyword/models/allbids-scrap-model.js @@ -0,0 +1,119 @@ +import browser from "../system/browser.js"; +import { extractModelId, extractNumber } from "../system/ultils.js"; +import { ScrapModel } from "./scrap-model.js"; + +export class AllbidsScrapModel extends ScrapModel { + action = async () => { + const urlsData = this.extractUrls(); + + for (let item of urlsData) { + await this.page.goto(item.url); + + const data = await this.getItemsInHtml(item); + + const results = this.filterItemByKeyword(item.keyword, data); + + this.results[item.keyword] = results; + + console.log({ results: this.results }); + } + }; + + async getPrice(url) { + const newPage = await browser.newPage(); // cần truyền 'url' từ bên ngoài nếu chưa có + + try { + await newPage.goto(`${this.web_bid.origin_url}${url}`, { + waitUntil: "domcontentloaded", // hoặc "networkidle2" nếu cần đợi AJAX + }); + + await newPage.waitForSelector("#bidPrefix > span.font-weight-bold", { + timeout: 10000, + }); + + const priceText = await newPage + .$eval("#bidPrefix > span.font-weight-bold", (el) => + el.textContent.trim() + ) + .catch(() => null); + + return extractNumber(priceText) || 0; + } catch (error) { + console.error(`Error getting price for ${url}:`, error); + return 0; + } finally { + await newPage.close(); + } + } + + getPriceByEl = async (elementHandle, model) => { + try { + const priceText = await elementHandle + .$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) => + el.textContent.trim() + ) + .catch(() => null); + + return extractNumber(priceText) || 0; + } catch (error) { + return 0; + } + }; + + getItemsInHtml = async (data) => { + await this.page.waitForSelector('input[name="searchText"]', { + timeout: 10000, + }); + + await this.page.type('input[name="searchText"]', data.keyword); + + await Promise.all([ + this.page.click( + "form .btn.btn-lg.btn-primary.waves-effect.allbids-cta-bid" + ), + this.page.waitForNavigation({ waitUntil: "networkidle0" }), // hoặc 'networkidle2' + ]); + + const elements = await this.page.$$("tbody > tr.row.ng-scope"); + + const results = []; + + for (const el of elements) { + const url = await el + .$eval(".col-md-5.col-lg-7.title > a", (el) => el.getAttribute("href")) + .catch(() => null); + + const model = extractModelId(url); + + const image_url = await el + .$eval(`#list${model} > div > img`, (img) => img.getAttribute("src")) + .catch(() => null); + + const name = await el + .$eval(`#list${model} > div:nth-child(1) > h3`, (el) => + el.textContent.trim() + ) + .catch(() => null); + + const priceText = await el + .$eval( + `#list${model} > div:nth-child(1) > div:nth-child(1) > span`, + (el) => el.textContent.trim() + ) + .catch(() => null); + + results.push({ + url, + image_url, + name, + keyword: data.keyword, + model, + current_price: extractNumber(priceText) || 0, + scrap_config_id: this.scrap_config_id, + }); + } + + console.log({ results }); + return results; + }; +} diff --git a/scrape-data-keyword/models/grays-scrap-model.js b/scrape-data-keyword/models/grays-scrap-model.js new file mode 100644 index 0000000..328f85d --- /dev/null +++ b/scrape-data-keyword/models/grays-scrap-model.js @@ -0,0 +1,74 @@ +import { extractModelId, extractNumber } from "../system/ultils.js"; +import { ScrapModel } from "./scrap-model.js"; + +export class GraysScrapModel extends ScrapModel { + action = async () => { + const urlsData = this.extractUrls(); + + for (let item of urlsData) { + await this.page.goto(item.url); + + const data = await this.getItemsInHtml(item); + + const results = this.filterItemByKeyword(item.keyword, data); + + this.results[item.keyword] = results; + } + + console.log({ results: this.results }); + }; + + getPriceByEl = async (elementHandle) => { + const selectors = [ + ".sc-ijDOKB.sc-bStcSt.ikmQUw.eEycyP", // Single product price + ".sc-ijDOKB.ikmQUw", // Multiple product price + ]; + + for (const selector of selectors) { + const priceText = await elementHandle + .$eval(selector, (el) => el.textContent.trim()) + .catch(() => null); + if (priceText) { + const price = extractNumber(priceText); + if (price) return price; + } + } + + return null; + }; + + getItemsInHtml = async (data) => { + const elements = await this.page.$$(".sc-102aeaf3-1.eYPitT > div"); + const results = []; + + for (const el of elements) { + const url = await el + .$eval(".sc-pKqro.sc-gFnajm.gqkMpZ.dzWUkJ", (el) => + el.getAttribute("href") + ) + .catch(() => null); + + const image_url = await el + .$eval("img.sc-gtJxfw.jbgdlx", (img) => img.getAttribute("src")) + .catch(() => null); + + const name = await el + .$eval(".sc-jlGgGc.dJRywx", (el) => el.textContent.trim()) + .catch(() => null); + + const current_price = await this.getPriceByEl(el); // Gọi hàm async được định nghĩa trong class + + results.push({ + url, + image_url, + name, + keyword: data.keyword, + model: extractModelId(url), + current_price, + scrap_config_id: this.scrap_config_id, + }); + } + + return results; + }; +} diff --git a/scrape-data-keyword/models/langtons-scrap-model.js b/scrape-data-keyword/models/langtons-scrap-model.js new file mode 100644 index 0000000..3e65b11 --- /dev/null +++ b/scrape-data-keyword/models/langtons-scrap-model.js @@ -0,0 +1,55 @@ +import { extractModelId, extractNumber } from "../system/ultils.js"; +import { ScrapModel } from "./scrap-model.js"; + +export class LangtonsScrapModel extends ScrapModel { + action = async () => { + const urlsData = this.extractUrls(); + + for (let item of urlsData) { + await this.page.goto(item.url); + + const data = await this.getItemsInHtml(item); + + const results = this.filterItemByKeyword(item.keyword, data); + + this.results[item.keyword] = results; + + console.log({ results: this.results }); + } + }; + + getItemsInHtml = async (data) => { + const elements = await this.page.$$(".row.product-grid.grid-view"); + const results = []; + + for (const el of elements) { + const url = await el + .$eval("a.js-pdp-link.pdp-link-anchor", (el) => el.getAttribute("href")) + .catch(() => null); + + const image_url = await el + .$eval("img.tile-image.loaded", (img) => img.getAttribute("src")) + .catch(() => null); + + const name = await el + .$eval(".link.js-pdp-link", (el) => el.textContent.trim()) + .catch(() => null); + + const current_price = await el + .$eval("div.max-bid-price.price", (el) => el.textContent.trim()) + .catch(() => null); + + results.push({ + url: `${this.web_bid.origin_url}${url}`, + image_url, + name, + keyword: data.keyword, + model: extractModelId(`${this.web_bid.origin_url}${url}`), + current_price: extractNumber(current_price), + scrap_config_id: this.scrap_config_id, + }); + } + + return results; + }; +} diff --git a/scrape-data-keyword/models/lawsons-scrap-model.js b/scrape-data-keyword/models/lawsons-scrap-model.js new file mode 100644 index 0000000..f595665 --- /dev/null +++ b/scrape-data-keyword/models/lawsons-scrap-model.js @@ -0,0 +1,95 @@ +import browser from "../system/browser.js"; +import { extractModelId, extractNumber } from "../system/ultils.js"; +import { ScrapModel } from "./scrap-model.js"; + +export class LawsonsScrapModel extends ScrapModel { + action = async () => { + const urlsData = this.extractUrls(); + + for (let item of urlsData) { + await this.page.goto(item.url); + + const data = await this.getItemsInHtml(item); + + const results = this.filterItemByKeyword(item.keyword, data); + + this.results[item.keyword] = results; + + console.log({ results: this.results }); + } + }; + + async getPrice(url) { + const newPage = await browser.newPage(); + + await newPage.setUserAgent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" + ); + + try { + await newPage.goto(`${this.web_bid.origin_url}${url}`, { + waitUntil: "domcontentloaded", // hoặc "networkidle2" nếu cần đợi AJAX + }); + + await newPage.waitForSelector("#bidPrefix > span.font-weight-bold", { + timeout: 10000, + }); + + const priceText = await newPage + .$eval("#bidPrefix > span.font-weight-bold", (el) => + el.textContent.trim() + ) + .catch(() => null); + + return extractNumber(priceText) || 0; + } catch (error) { + console.error(`Error getting price for ${url}:`, error); + return 0; + } finally { + await newPage.close(); + } + } + + getItemsInHtml = async (data) => { + await this.page.waitForSelector(".row.row-spacing > .lot-container", { + timeout: 10000, + }); + + const elements = await this.page.$$(".row.row-spacing > .lot-container"); + const results = []; + + for (const el of elements) { + const url = await el + .$eval("aside.search-lot--content.text-left > a", (el) => + el.getAttribute("href") + ) + .catch(() => null); + + const image_url = await el + .$eval("figure.text-center.imgContainer img", (img) => + img.getAttribute("src") + ) + .catch(() => null); + + const name = await el + .$eval(".font-weight-normal.text-grey.title", (el) => + el.textContent.trim() + ) + .catch(() => null); + + const current_price = await this.getPrice(url); + + results.push({ + url: `${this.web_bid.origin_url}${url}`, + image_url, + name, + keyword: data.keyword, + model: extractModelId(`${this.web_bid.origin_url}${url}`), + current_price: current_price, + scrap_config_id: this.scrap_config_id, + }); + } + + return results; + }; +} diff --git a/scrape-data-keyword/models/pickles-scrap-model.js b/scrape-data-keyword/models/pickles-scrap-model.js new file mode 100644 index 0000000..7874cf4 --- /dev/null +++ b/scrape-data-keyword/models/pickles-scrap-model.js @@ -0,0 +1,108 @@ +import browser from "../system/browser.js"; +import { extractModelId, extractNumber } from "../system/ultils.js"; +import { ScrapModel } from "./scrap-model.js"; + +export class PicklesScrapModel extends ScrapModel { + action = async () => { + const urlsData = this.extractUrls(); + + for (let item of urlsData) { + await this.page.goto(item.url); + + const data = await this.getItemsInHtml(item); + + const results = this.filterItemByKeyword(item.keyword, data); + + this.results[item.keyword] = results; + + console.log({ results: this.results }); + } + }; + + async getPrice(url) { + const newPage = await browser.newPage(); // cần truyền 'url' từ bên ngoài nếu chưa có + + try { + await newPage.goto(`${this.web_bid.origin_url}${url}`, { + waitUntil: "domcontentloaded", // hoặc "networkidle2" nếu cần đợi AJAX + }); + + await newPage.waitForSelector("#bidPrefix > span.font-weight-bold", { + timeout: 10000, + }); + + const priceText = await newPage + .$eval("#bidPrefix > span.font-weight-bold", (el) => + el.textContent.trim() + ) + .catch(() => null); + + return extractNumber(priceText) || 0; + } catch (error) { + console.error(`Error getting price for ${url}:`, error); + return 0; + } finally { + await newPage.close(); + } + } + + getPriceByEl = async (elementHandle, model) => { + try { + const priceText = await elementHandle + .$eval(`#ps-bg-buy-btn-${model} .pds-button-label`, (el) => + el.textContent.trim() + ) + .catch(() => null); + + return extractNumber(priceText) || 0; + } catch (error) { + return 0; + } + }; + + getItemsInHtml = async (data) => { + await this.page.waitForSelector( + ".content-wrapper_contentgridwrapper__3RCQZ > div.column", + { + timeout: 10000, + } + ); + + const elements = await this.page.$$( + ".content-wrapper_contentgridwrapper__3RCQZ > div.column" + ); + const results = []; + + for (const el of elements) { + const url = await el + .$eval("main > a", (el) => el.getAttribute("href")) + .catch(() => null); + + const image_url = await el + .$eval("div > div:first-child > div > div:first-child > img", (img) => + img.getAttribute("src") + ) + .catch(() => null); + + const name = await el + .$eval("header > h2:first-of-type", (el) => el.textContent.trim()) + .catch(() => null); + + const model = extractModelId(`${this.web_bid.origin_url}${url}`); + + const current_price = await this.getPriceByEl(el, model); + + results.push({ + url: `${this.web_bid.origin_url}${url}`, + image_url, + name, + keyword: data.keyword, + model, + current_price: current_price, + scrap_config_id: this.scrap_config_id, + }); + } + + return results; + }; +} diff --git a/scrape-data-keyword/models/scrap-model.js b/scrape-data-keyword/models/scrap-model.js new file mode 100644 index 0000000..92152a8 --- /dev/null +++ b/scrape-data-keyword/models/scrap-model.js @@ -0,0 +1,35 @@ +export class ScrapModel { + constructor({ keywords, search_url, scrap_config_id, page, web_bid }) { + this.keywords = keywords; + this.search_url = search_url; + this.scrap_config_id = scrap_config_id; + this.results = {}; + this.page = page; + this.web_bid = web_bid; + } + + buildUrlWithKey(rawUrl, keyword) { + return rawUrl.replaceAll("{{keyword}}", keyword); + } + + extractUrls() { + const keywordList = this.keywords.split(", "); + if (keywordList.length <= 0) return []; + + return keywordList.map((keyword) => ({ + url: this.buildUrlWithKey(this.search_url, keyword), + keyword, + })); + } + + filterItemByKeyword(keyword, data) { + return data.filter((item) => + item.name.toLowerCase().includes(keyword.toLowerCase()) + ); + } + + // Phần này bạn cần truyền từ bên ngoài khi khởi tạo hoặc kế thừa class: + action = async (page) => {}; + getInfoItems = (data) => []; + getItemsInHtml = async (data) => []; +} diff --git a/scrape-data-keyword/package-lock.json b/scrape-data-keyword/package-lock.json new file mode 100644 index 0000000..1ead6fe --- /dev/null +++ b/scrape-data-keyword/package-lock.json @@ -0,0 +1,1876 @@ +{ + "name": "scrape-data-keyword", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "scrape-data-keyword", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "axios": "^1.8.2", + "dotenv": "^16.4.7", + "lodash": "^4.17.21", + "puppeteer": "^24.4.0", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.5", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.5.tgz", + "integrity": "sha512-eifa0o+i8dERnngJwKrfp3dEq7ia5XFyoqB17S4gK8GhsQE4/P8nxOfQSE0zQHxzzLo/cmF+7+ywEQ7wK7Fb+w==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.1", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.2", + "tar-fs": "^3.0.8", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.15.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.21.tgz", + "integrity": "sha512-EV/37Td6c+MgKAbkcLG6vqZ2zEYHD7bvSrzqqs2RIhbA6w3x+Dqz8MZM3sP6kGTeLrdoOgKZe+Xja7tUB2DNkQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.9.0.tgz", + "integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.1.5.tgz", + "integrity": "sha512-1zccWBMypln0jEE05LzZt+V/8y8AQsQQqxtklqaIyg5nu6OAYFhZxPXinJTSG+kU5qyNmeLgcn9AW7eHiCHVLA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.1.tgz", + "integrity": "sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chromium-bidi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-5.1.0.tgz", + "integrity": "sha512-9MSRhWRVoRPDG0TgzkHrshFSJJNZzfY5UFqUMuksg7zL1yoZIZ3jLB0YAgHclbiAxPI86pBnwDX1tbzoiV8aFw==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "license": "MIT", + "dependencies": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1439962", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1439962.tgz", + "integrity": "sha512-jJF48UdryzKiWhJ1bLKr7BFWUQCEIT5uCNbDLqkQJBtkFxYzILJH44WN0PDKMIlGDN7Utb8vyUY85C3w4R/t2g==", + "license": "BSD-3-Clause" + }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "license": "MIT", + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", + "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "license": "MIT", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT" + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "license": "MIT", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/merge-deep": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz", + "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", + "license": "MIT", + "dependencies": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "license": "MIT", + "dependencies": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.9.0.tgz", + "integrity": "sha512-L0pOtALIx8rgDt24Y+COm8X52v78gNtBOW6EmUcEPci0TYD72SAuaXKqasRIx4JXxmg2Tkw5ySKcpPOwN8xXnQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.5", + "chromium-bidi": "5.1.0", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1439962", + "puppeteer-core": "24.9.0", + "typed-query-selector": "^2.12.0" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.9.0.tgz", + "integrity": "sha512-HFdCeH/wx6QPz8EncafbCqJBqaCG1ENW75xg3cLFMRUoqZDgByT6HSueiumetT2uClZxwqj0qS4qMVZwLHRHHw==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.5", + "chromium-bidi": "5.1.0", + "debug": "^4.4.1", + "devtools-protocol": "0.0.1439962", + "typed-query-selector": "^2.12.0", + "ws": "^8.18.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-extra": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz", + "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==", + "license": "MIT", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "deepmerge": "^4.2.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "@types/puppeteer": "*", + "puppeteer": "*", + "puppeteer-core": "*" + }, + "peerDependenciesMeta": { + "@types/puppeteer": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "puppeteer-core": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz", + "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==", + "license": "MIT", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + }, + "engines": { + "node": ">=9.11.2" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-stealth": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz", + "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-preferences": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-data-dir": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz", + "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "fs-extra": "^10.0.0", + "puppeteer-extra-plugin": "^3.2.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-preferences": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz", + "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.2.2", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-data-dir": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", + "license": "MIT", + "dependencies": { + "is-buffer": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.4.tgz", + "integrity": "sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ==", + "license": "MIT", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", + "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT", + "optional": true + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.18.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", + "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/zod": { + "version": "3.25.13", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.13.tgz", + "integrity": "sha512-Q8mvk2iWi7rTDfpQBsu4ziE7A6AxgzJ5hzRyRYQkoV3A3niYsXVwDaP1Kbz3nWav6S+VZ6k2OznFn8ZyDHvIrg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/scrape-data-keyword/package.json b/scrape-data-keyword/package.json new file mode 100644 index 0000000..f489048 --- /dev/null +++ b/scrape-data-keyword/package.json @@ -0,0 +1,22 @@ +{ + "name": "scrape-data-keyword", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node index.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "module", + "dependencies": { + "axios": "^1.8.2", + "dotenv": "^16.4.7", + "lodash": "^4.17.21", + "puppeteer": "^24.4.0", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2" + } +} diff --git a/scrape-data-keyword/services/scrap-configs-service.js b/scrape-data-keyword/services/scrap-configs-service.js new file mode 100644 index 0000000..fa7ba78 --- /dev/null +++ b/scrape-data-keyword/services/scrap-configs-service.js @@ -0,0 +1,56 @@ +import { AllbidsScrapModel } from "../models/allbids-scrap-model.js"; +import { GraysScrapModel } from "../models/grays-scrap-model.js"; +import { LangtonsScrapModel } from "../models/langtons-scrap-model.js"; +import { LawsonsScrapModel } from "../models/lawsons-scrap-model.js"; +import { PicklesScrapModel } from "../models/pickles-scrap-model.js"; + +export class ScrapConfigsService { + static scrapModel(scrapConfig, page) { + switch (scrapConfig.web_bid.origin_url) { + case "https://www.grays.com": { + return new GraysScrapModel({ + ...scrapConfig, + scrap_config_id: scrapConfig.id, + page: page, + }); + } + case "https://www.langtons.com.au": { + return new LangtonsScrapModel({ + ...scrapConfig, + scrap_config_id: scrapConfig.id, + page: page, + }); + } + case "https://www.lawsons.com.au": { + return new LawsonsScrapModel({ + ...scrapConfig, + scrap_config_id: scrapConfig.id, + page: page, + }); + } + case "https://www.pickles.com.au": { + return new PicklesScrapModel({ + ...scrapConfig, + scrap_config_id: scrapConfig.id, + page: page, + }); + } + case "https://www.allbids.com.au": { + return new AllbidsScrapModel({ + ...scrapConfig, + scrap_config_id: scrapConfig.id, + page: page, + }); + } + default: { + return null; + } + } + } + + static scrapModels(data, page) { + return data + .map((item) => this.scrapModel(item, page)) + .filter((item) => !!item); + } +} diff --git a/scrape-data-keyword/system/axios.js b/scrape-data-keyword/system/axios.js new file mode 100644 index 0000000..ae2d18a --- /dev/null +++ b/scrape-data-keyword/system/axios.js @@ -0,0 +1,11 @@ +import ax from "axios"; + +const axios = ax.create({ + // baseURL: 'http://172.18.2.125/api/v1/', + baseURL: process.env.BASE_URL, + headers: { + Authorization: process.env.CLIENT_KEY, + }, +}); + +export default axios; diff --git a/scrape-data-keyword/system/browser.js b/scrape-data-keyword/system/browser.js new file mode 100644 index 0000000..8aeb541 --- /dev/null +++ b/scrape-data-keyword/system/browser.js @@ -0,0 +1,39 @@ +// import puppeteer from 'puppeteer'; +import puppeteer from "puppeteer-extra"; +import StealthPlugin from "puppeteer-extra-plugin-stealth"; + +puppeteer.use(StealthPlugin()); +const browser = await puppeteer.launch({ + headless: process.env.ENVIRONMENT === "prod" ? "new" : false, + // userDataDir: CONSTANTS.PROFILE_PATH, // Thư mục lưu profile + timeout: 60000, + args: [ + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + "--disable-software-rasterizer", + "--disable-background-networking", + "--disable-sync", + "--mute-audio", + "--no-first-run", + "--no-default-browser-check", + "--ignore-certificate-errors", + "--start-maximized", + "--disable-site-isolation-trials", // Tắt sandbox riêng cho từng site + "--memory-pressure-off", // Tắt cơ chế bảo vệ bộ nhớ + "--disk-cache-size=0", // Không dùng cache để giảm bộ nhớ + "--enable-low-end-device-mode", // Kích hoạt chế độ tiết kiệm RAM + "--disable-best-effort-tasks", // Tắt tác vụ không quan trọng + "--disable-accelerated-2d-canvas", // Không dùng GPU để vẽ canvas + "--disable-threaded-animation", // Giảm animation chạy trên nhiều thread + "--disable-threaded-scrolling", // Tắt cuộn trang đa luồng + "--disable-logging", // Tắt log debug + "--blink-settings=imagesEnabled=false", // Không tải hình ảnh, + "--disable-background-timer-throttling", // Tránh việc throttling các timer khi chạy nền. + "--disable-webrtc", + "--disable-ipc-flooding-protection", // Nếu có extension cần IPC, cái này giúp tối ưu. + ], +}); + +export default browser; diff --git a/scrape-data-keyword/system/ultils.js b/scrape-data-keyword/system/ultils.js new file mode 100644 index 0000000..61094bd --- /dev/null +++ b/scrape-data-keyword/system/ultils.js @@ -0,0 +1,47 @@ +export function extractNumber(str) { + if (typeof str !== "string") return null; + + const cleaned = str.replace(/,/g, ""); + const match = cleaned.match(/\d+(\.\d+)?/); + return match ? parseFloat(match[0]) : null; +} + +export function extractModelId(url) { + try { + switch (extractDomain(url)) { + case "https://www.grays.com": { + const match = url.match(/\/lot\/([\d-]+)\//); + return match ? match[1] : null; + } + case "https://www.langtons.com.au": { + const match = url.match(/auc-var-\d+/); + return match[0]; + } + case "https://www.lawsons.com.au": { + const match = url.split("_"); + return match ? match[1] : null; + } + case "https://www.pickles.com.au": { + const model = url.split("/").pop(); + return model ? model : null; + } + case "https://www.allbids.com.au": { + const match = url.match(/-(\d+)(?:[\?#]|$)/); + return match ? match[1] : null; + } + default: + return null; + } + } catch (error) { + return null; + } +} + +export function extractDomain(url) { + try { + const parsedUrl = new URL(url); + return parsedUrl.origin; + } catch (error) { + return null; + } +}