joseph
/
Log_service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update logic check SN RAM

This commit is contained in:
Joseph Le 2026-01-19 13:43:35 +11:00
parent 18daeb857b
commit fd6568ddd1
10 changed files with 4771 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitconfig
View File

@ -10,3 +10,7 @@
directory = /home/Log_service/
directory = /home/Log_service/
directory = /home/Log_service/
directory = /home/Log_service/
directory = /home/Log_service/
directory = /home/Log_service/
directory = /home/Log_service/

74
app/Controllers/Http/ErpsController.ts
View File

@ -6,7 +6,7 @@ import LogDetectFile from "App/Models/LogDetectFile";
const path = require("path");
const BASE_URL = Env.get("BASE_URL_LOG");
const BASE_URL_AUTO = `${BASE_URL}/AUTOlog/`;
const BASE_URL_AUTO = `${BASE_URL}/`;
export default class ErpsController {
/**
* Controller do tim cac serial number trong cac log trong khoang thoi gian xac dinh
@ -26,12 +26,12 @@ export default class ErpsController {
console.log("check!!!!!!!!!!!!!!!!!!!!!")
let data1 = response.data
.split("\n")
.filter((i) => i.search("<a href") !== -1 && i.search(".log") !== -1);
.split("\n")
.filter((i) => i.search("<a href") !== -1 && i.search(".log") !== -1);
const arrayLine = data1;
arrayLine.map(async(u) => {
arrayLine.map(async (u) => {
let fileName = u.slice(u.search("<a ") + 9, u.search("</a>")).split(">")[1];
if (
@ -41,10 +41,10 @@ export default class ErpsController {
listLog.push(
(BASE_URL + "/") +
fileName +
" "
" "
);
const record = await LogDetectFile.firstOrCreate(
const record = await LogDetectFile.firstOrCreate(
{ file_name: fileName },
{ file_name: fileName }
);
@ -78,8 +78,7 @@ export default class ErpsController {
return null;
};
const extractInfoFromLine = (line, index, fName, lines) => {
const cleanedLine = line.replace(/\r/g, "");
const extractInfoFromLine = (cleanedLine, index, fName, lines, brand, PID) => {
const results: any = [];
const getWarehouse = (fileName) =>
@ -98,19 +97,21 @@ export default class ErpsController {
const SN = extractSN(
parts.find((p) => p.includes("SN:"))?.split(":")[1] ?? ""
);
const PID = extractSN(
PID = extractSN(
parts.find((p) => p.includes("PID:"))?.split(":")[1] ?? ""
);
const VID = extractSN(
parts.find((p) => p.includes("VID:"))?.split(":")[1] ?? ""
);
if (SN && SN !== "N/A" && SN.length > 4) {
results.push({
PID,
VID,
SN,
line: [index + 1],
brand,
fileName: fName,
warehouse: getWarehouse(fName),
});
@ -122,10 +123,11 @@ export default class ErpsController {
const PCB_SN = extractSN(cleanedLine.split(":")[1]);
if (PCB_SN) {
results.push({
PID: "",
PID,
VID: "",
SN: PCB_SN,
line: [index + 1],
brand,
fileName: fName,
warehouse: getWarehouse(fName),
});
@ -137,10 +139,11 @@ export default class ErpsController {
const PCB_SN = extractSN(cleanedLine.split(":")[1]);
if (PCB_SN) {
results.push({
PID: "",
PID,
VID: "",
SN: PCB_SN.trim(),
line: [index + 1],
brand,
fileName: fName,
warehouse: getWarehouse(fName),
});
@ -151,10 +154,11 @@ export default class ErpsController {
const PCB_SN = extractSN(cleanedLine.split(":")[1]);
if (PCB_SN) {
results.push({
PID: "",
PID,
VID: "",
SN: PCB_SN,
line: [index + 1],
brand,
fileName: fName,
warehouse: getWarehouse(fName),
});
@ -166,16 +170,19 @@ export default class ErpsController {
const PBID = extractSN(cleanedLine.split(" ").pop());
if (PBID?.length >= 8) {
results.push({
PID: "",
PID,
VID: "",
SN: PBID,
line: [index + 1],
brand,
fileName: fName,
warehouse: getWarehouse(fName),
});
}
}
return results;
};
@ -200,7 +207,7 @@ export default class ErpsController {
return [];
}
const fileContents = await Promise.all(
urls.map((u) => fetchWithRetry(u?.split(".log")[0]+".log"))
urls.map((u) => fetchWithRetry(u?.split(".log")[0] + ".log"))
);
let report: any = [];
@ -211,9 +218,33 @@ export default class ErpsController {
const fName = path.basename(urls[index] ?? "").trim();
const output = [];
//Vendor Description: --> brand
let currentBrand = ""
let currentPID = ""
lines.forEach((line, i) => {
const lineResult = extractInfoFromLine(line, i, fName, lines);
const cleanedLine = line.replace(/\r/g, "");
if (cleanedLine.includes("Vendor Description")) {
currentBrand = cleanedLine.split("Vendor Description:")[1]?.trim();
}
if (cleanedLine.includes("Vendor Part Number")) {
// console.log("Brand: ", cleanedLine)
// console.log("Brand1: ", cleanedLine.split("Vendor Part Number:")[1])
currentPID = cleanedLine.split("Vendor Part Number:")[1]?.trim();
}
const lineResult = extractInfoFromLine(cleanedLine, i, fName, lines, currentBrand, currentPID);
// Nếu dòng này tạo ra SN nào đó -> coi như brand đã được dùng xong
if (lineResult.length > 0 && currentBrand) {
currentBrand = "";
}
if (lineResult.length > 0 && currentPID) {
currentPID = "";
}
mergeLines(output, lineResult);
});
@ -252,8 +283,7 @@ export default class ErpsController {
const { fileName, line, range } = request.all();
try {
let fName =
fileName.search("AUTO") !== -1 ? "AUTOlog/" + fileName : fileName;
let fName = fileName;
const res = await axios.get(BASE_URL + "/" + fName);
const arrayLine = res?.data?.split("\n");
@ -272,13 +302,13 @@ export default class ErpsController {
}
}
public async store({ request, response }: HttpContextContract) {}
public async store({ request, response }: HttpContextContract) { }
public async show({}: HttpContextContract) {}
public async show({ }: HttpContextContract) { }
public async edit({}: HttpContextContract) {}
public async edit({ }: HttpContextContract) { }
public async update({}: HttpContextContract) {}
public async update({ }: HttpContextContract) { }
public async destroy({}: HttpContextContract) {}
public async destroy({ }: HttpContextContract) { }
}

39
app/utils/checkIndexSN.ts
View File

@ -7,6 +7,7 @@ type OutputItem = {
PID: string;
VID: string;
SN: string;
brand: string;
line: number[];
fileName: string;
warehouse: "US" | "AU";
@ -22,7 +23,8 @@ export const checkIndexSN = async (
const output: OutputItem[] = [];
if (!arrayLine) return;
let PID = ""
let brand = ""
for (let i = 0; i < arrayLine.length; i++) {
const line = arrayLine[i];
const SN = line
@ -30,6 +32,21 @@ export const checkIndexSN = async (
?.trim()
.replace(/[!@#$%^&*()_+{}\[\]:;<>,.?~\\/]/g, "");
if (line.includes("Vendor Part Number")) {
PID = line.split("Vendor Part Number:")[1]?.trim();
}
//Vendor Description: --> brand
if (line.includes("Vendor Description") && i >= beginLine) {
const rawBrand = line.split("Vendor Description")[1]?.split(":")[1]
?.replace("\r", "")
?.trim()
?.replace(/[!@#$%^&*()_+{}\[\]:;<>,.?~\\/]/g, "");
if (!rawBrand) continue;
brand = rawBrand.includes(" ") ? rawBrand.split(" ")[0] : rawBrand;
console.log(brand)
}
// Pattern: PID + SN
if (
line.includes("PID:") &&
@ -41,7 +58,7 @@ export const checkIndexSN = async (
i >= beginLine
) {
const existingIndex = output.findIndex((u) => u.SN === SN);
const PID = line?.split("VID:")[0]
PID = line?.split("VID:")[0]
?.slice(line?.split("VID:")[0]?.search("PID"))
?.split(":")[1]
?.split(",")[0]
@ -62,6 +79,7 @@ export const checkIndexSN = async (
VID,
SN: SN.includes(" ") ? SN.split(" ")[0] : SN,
line: [i + 1],
brand: brand,
fileName: nameF,
warehouse:
(nameF.includes("-US") ||
@ -90,10 +108,11 @@ export const checkIndexSN = async (
existing.line.push(i + 1);
} else {
output.push({
PID: "",
PID,
VID: "",
SN,
line: [i + 1],
brand: brand,
fileName: nameF,
warehouse:
(nameF.includes("-US") ||
@ -122,10 +141,11 @@ export const checkIndexSN = async (
existing.line.push(i + 1);
} else {
output.push({
PID: "",
PID,
VID: "",
SN,
line: [i + 1],
brand: brand,
fileName: nameF,
warehouse:
(nameF.includes("-US") ||
@ -137,7 +157,6 @@ export const checkIndexSN = async (
});
}
}
//System serial num
if (line.includes("System serial num") && i >= beginLine) {
const rawSN = line.split("System serial num")[1]?.split(":")[1]
@ -154,10 +173,11 @@ export const checkIndexSN = async (
existing.line.push(i + 1);
} else {
output.push({
PID: "",
PID,
VID: "",
SN,
line: [i + 1],
brand: brand,
fileName: nameF,
warehouse:
(nameF.includes("-US") ||
@ -187,10 +207,11 @@ export const checkIndexSN = async (
existing.line.push(i + 1);
} else {
output.push({
PID: "",
PID,
VID: "",
SN,
line: [i + 1],
brand: brand,
fileName: nameF,
warehouse:
(nameF.includes("-US") ||
@ -205,9 +226,7 @@ export const checkIndexSN = async (
}
const pattern = /[\x00-\x20\x7F]/g;
const invalidPIDItems = output.filter(
(i) => !i.PID.match(pattern)
);
const invalidPIDItems = output
const fileName =

13
app/utils/google.json Normal file
View File

@ -0,0 +1,13 @@
{
"type": "service_account",
"project_id": "export-items-sold-out",
"private_key_id": "0020435b2b32237dc52aaabc1829fecacf57b6be",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCU4x7urMjxRfZ3\nX3BOTaXOh0epkImJuFuYCAEvLFmECqTOJUuTMqTQaknVde53d4H7WNwuT9kqZLK7\nkyVjRSww2H4NFMAt6eWa9K/vjOBJgFrJVdp31IyGBs5IoD47ENV6ZCNk79K2+wRn\n2Vg9jfkHBGgKPVNrbYh4oK6e7wrJ99En3SJvkmOJu1IbVe55/vo24aJQtaxBNwfR\nNWb81EzmQWOnj4iYkbD52WNiWwYhFBdpGr0mAadhxrcERYM/EcLB8dMYJHsA73sm\nXedUv3ocGUuf3U8WDxcAvhkSq7TMk6F+0YcoB1XWZBCbSsBWcR31RQHaVZ3Kchu7\nT51hvmbLAgMBAAECggEAQBeVVUFkLxjjBHg9LDgYzFNH8iKaUoLeSxAHEnuQ/Vpn\nn6ZGZlNCx8U5i3587Uiw17Zk8jrFTY6TslOhoJ5Ca7SGbrMkF5+8i3wR0JlgY8Ox\nUUJ4DWWeKIP9gR4aK+sPOuSop+blWoesWdgiOD1Jmk+3SdEke/Zm7SaehSR6DH7N\nbOKUD2M71FOWlxU5V10J+CAfVgAvC4rl5TD59LijyzVUi46XxvizDBfqPWuskEK0\np35uf/kHKhvc7O8wolGHAECuhQ7btXSyS/A19aTfIquc5PzIyV48Kif+PBr7VAse\nAADf4k0Bx6q+8suA2ftG41jzNYhQCo8osslJanaGgQKBgQDNDIfgzjSjTuq2u8NU\nCCXxRYddVL6zgdeU0COJ23YnXp/6BX2K5vvL3sTU8Oz0coLFuoVNnZ6WKW1hHTAS\nUAxbfl6FtBKh+MsGmT7TWDvkog+Uxr85GNgmwZv2m5Kmm3T1sftYj5kWLWTgs9u4\nMZRfo6B2hSUYF3ELG6zPiQvkWwKBgQC54g6UCscp0hFxSASSIKFz5fkJJe2PtSfn\nAeDoS1bh8meqvbxyJ8NR6nPHDb356IDw5S6gJf28JhB18gILZftWVWZg1IENu0x+\nILeK/HO3F25w+MDAT0D6jSIhjpIcGjHH5tXR8ArndAkJO/I/TSZ9gXJlPC993pE9\n53LtUWJSUQKBgQCE1l5ctBoDxkdmdIOZN+IkyS9ltlYJAs//10t+rs9/t9yZ496u\nEvAC5mHLtf08a8bug+SCIN+xVktHA9q1smgfCeIAamU7y2rz7ml2D45VkuvXLV6K\nhd7aVMlOXQEHsGR4G4LCn9zEMkpNDBgG3w/JWb7FqYIbdmxEjvxIQveSSwKBgGlh\nUOZ43cRFxYG1VHILcbyk6fRXvkA0cXSzSfAnlg/uPvs7KbSF+mNcSVrV3ZfqSf9H\nzpEYOKsL4VUcaDBSFpUxQSMwFiHUx17R6Xwv9uIbtI7zV0MvKyVgEbN4gjjQWNx9\nb0bfkS76R40moQVmFaYdVt8xqbLwFONFbZBBDT6hAoGBALdoPSCWDQNkWYrHm5nN\nFCqbF8JjDoolLh+KkiA9lphT5wxJMf35JIhUodEH1WFoLHNjSP1jCNk0UVj/4/aS\nrXphqiCmZNriqfec7B9eWsR7h482rl24DaOHVYjeBR6uqGUVJKVuQ/KkpKC15Rbm\nRYcVX/gCpP9/QNYPsPnBriEG\n-----END PRIVATE KEY-----\n",
"client_email": "export-file-service@export-items-sold-out.iam.gserviceaccount.com",
"client_id": "111585688432424787777",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/export-file-service%40export-items-sold-out.iam.gserviceaccount.com",
"universe_domain": "googleapis.com"
}

708
app/utils/runtimeCheckLogs.ts
View File

@ -1,9 +1,11 @@
import Env from "@ioc:Adonis/Core/Env";
import Database from "@ioc:Adonis/Lucid/Database";
import KeyValue from "App/Models/KeyValue";
import LogDetectFile from "App/Models/LogDetectFile";
import LogReport from "App/Models/LogReport";
import chokidar from "chokidar";
import fs from "fs";
import readline from "readline";
import { DateTime } from "luxon";
import moment from "moment";
import { checkIndexSN } from "./checkIndexSN";
@ -11,423 +13,371 @@ import { getListLineByItem } from "./getListLineByItem";
import { sendMessToZulip } from "./sendMessToZulip";
import { checkSpecialVersion } from "./helper";
export async function runtimeCheckLogs(folderPath: string) {
/**
* ======================================================
* CONSTANTS
* ======================================================
*/
const MAX_LINES_PER_BATCH = 5000; // Process max 5k lines at a time
const MAX_INSERTS_PER_BATCH = 500; // Insert max 500 records per transaction
/**
* ======================================================
* KEY VALUE CACHE (GLOBAL)
* ======================================================
*/
let keyValueCache: Record<string, string[]> | null = null;
let keyValueCacheAt = 0;
const CACHE_TTL = 60_000;
async function getKeyValueCached() {
if (keyValueCache && Date.now() - keyValueCacheAt < CACHE_TTL) {
return keyValueCache;
}
const rows = await KeyValue.query().select("key", "value");
const map: Record<string, string[]> = {};
for (const r of rows) {
if (!map[r.key]) map[r.key] = [];
map[r.key].push(r.value);
}
keyValueCache = map;
keyValueCacheAt = Date.now();
return map;
}
/**
* ======================================================
* FILE LOCK + DEBOUNCE
* ======================================================
*/
const fileLocks = new Set<string>();
const debounceMap = new Map<string, NodeJS.Timeout>();
async function safeProcessFileChange(filePath: string) {
if (fileLocks.has(filePath)) return;
fileLocks.add(filePath);
try {
let cacheFile: any = [];
let fileList = Array();
let fileList_old = Array();
// Function to update the list of files
async function updateFileList() {
//get date now
let dateNow = moment(Date.now()).format("YYYY/MM/DD");
const dateCheck = moment(Date.now()).format("YYYYMMDD");
//Get list file in folder
fileList = fs.readdirSync(folderPath);
//Get date modified
fileList.map((file) => {
const filePath = `${folderPath}/${file}`;
if (file?.split(".")[filePath.split(".").length - 1] === "log") {
const stats = fs.statSync(filePath);
await processFileChange(filePath);
} catch (err) {
console.error("processFileChange error:", err);
} finally {
fileLocks.delete(filePath);
}
}
//scan file today
if (dateNow !== moment(stats.mtime).format("YYYY/MM/DD")) {
fileList = fileList.filter((i) => i !== file && i.includes(dateCheck));
}
}
});
console.log(fileList)
}
/**
* ======================================================
* STREAM-BASED FILE READER (MEMORY EFFICIENT)
* ======================================================
*/
async function readLinesFromPosition(
filePath: string,
startLine: number,
maxLines: number = MAX_LINES_PER_BATCH
): Promise<Array<{ lineNo: number; content: string }>> {
return new Promise((resolve, reject) => {
const lines: Array<{ lineNo: number; content: string }> = [];
let currentLine = 0;
// Watch the folder for new files
const folderWatcher = chokidar.watch(folderPath);
folderWatcher.on("add", async (filePath) => {
//import log new file
//check format file name
if (
filePath?.split(".")[filePath.split(".").length - 1] === "log" &&
filePath.split("/")[filePath.split("/").length - 1]?.split("-")[0] ===
//localhost
// filePath.split("\\")[filePath.split("\\").length - 1]?.split("-")[0] ===
moment(Date.now()).format("YYYYMMDD").toString()
) {
//add information file to database
console.log("New file added: ", filePath);
await LogDetectFile.firstOrCreate(
{ file_name: filePath.split("/")[filePath.split("/").length - 1] },
{ file_name: filePath.split("/")[filePath.split("/").length - 1] }
);
//localhost
// await LogDetectFile.firstOrCreate(
// { file_name: filePath.split("\\")[filePath.split("\\").length - 1] },
// { file_name: filePath.split("\\")[filePath.split("\\").length - 1] }
// );
fileList_old = fileList;
updateFileList();
watchFilesInList();
}
// updateFile;
const stream = fs.createReadStream(filePath, { encoding: "utf8" });
const rl = readline.createInterface({
input: stream,
crlfDelay: Infinity,
});
// Watch for changes in the files listed
async function watchFilesInList() {
try {
//only check new file ---> fileList - fileList_old = new file
let listFileWatch = fileList
?.filter(
(i) => fileList_old.includes(i) === false && i.includes(".log")
)
.map((file) => folderPath + "/" + file);
const watcher = chokidar.watch(listFileWatch, {
persistent: true,
usePolling: true,
interval: 300000,
});
rl.on("line", (line) => {
currentLine++;
if (currentLine > startLine) {
lines.push({ lineNo: currentLine, content: line });
watcher.setMaxListeners(200);
// ✅ FIX: Stop reading after maxLines to prevent memory overflow
if (lines.length >= maxLines) {
rl.close();
stream.destroy();
}
}
});
watcher.on("change", async (path) => {
// fs.watchFile(filePath,{ interval: 15000 },
// async (eventType) => {
//check special item, extra RAM, error in log
const fileName = path.split("/")[path.split("/").length - 1];
// const fileName = path.split("\\")[path.split("\\").length - 1];
const filePath = path;
let lines: any = [];
const today = DateTime.now().toFormat("yyyy-MM-dd");
let allFile = await LogDetectFile.query().whereRaw(
`DATE(created_at) = ?`,
[today]
);
// let allReport = await LogReport.all();
let allValue = await KeyValue.all();
const allReport = await LogReport.query().whereRaw(
`DATE(created_at) = ?`,
[today]
);
rl.on("close", () => resolve(lines));
rl.on("error", reject);
stream.on("error", reject);
});
}
cacheFile = cacheFile.concat(
allFile.filter(
(obj2) => !cacheFile.some((obj1) => obj1.id_ldf === obj2.id_ldf)
)
);
/**
* ======================================================
* CORE LOGIC (FIXED FOR STACK OVERFLOW)
* ======================================================
*/
async function processFileChange(filePath: string) {
const fileName = filePath.split("/").pop()!;
//get information file
let fileDetect = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
const fileDetect = await LogDetectFile
.query()
.where("file_name", fileName)
.first();
let logsDetect = allReport?.filter(
(i) => i.$original.id_file === fileDetect?.id_ldf
);
//get the last line detected
let lastLine = Math.max(...logsDetect.map((obj) => obj.line));
if (!fileDetect) return;
//get content file in local
let contentFile = await fs
.readFileSync(filePath)
.toString()
?.split("\n");
const existedReports = await LogReport
.query()
.select("line", "detected_content")
.where("id_file", fileDetect.id_ldf);
//get index SN and send to ERP
const existedSet = new Set(
existedReports.map(r => `${r.line}-${r.detected_content}`)
);
checkIndexSN(contentFile, lastLine, fileName);
let lastLine = Math.max(0, ...existedReports.map(r => r.line));
//get list item to check
let listKeyValues = allValue.filter(
(i) =>
i.$original.key === "MODEL_SPECIAL" ||
i.$original.key === "CATCH_FAULTY"
);
const keyMap = await getKeyValueCached();
const MODEL_SPECIAL = keyMap.MODEL_SPECIAL || [];
const CATCH_FAULTY = keyMap.CATCH_FAULTY || [];
const EXCLUDE_ERR = keyMap.EXCLUDE_ERR || [];
//get list exclude error
let listExcludeErr = allValue
.filter((i) => i.$original.key === "EXCLUDE_ERR")
.map((obj) => obj.$original.value);
// ✅ FIX: Combine arrays safely (avoid spread operator with large arrays)
const keysToCheck: string[] = [];
for (const key of MODEL_SPECIAL) keysToCheck.push(key);
for (const key of CATCH_FAULTY) keysToCheck.push(key);
//get list item special
let listExtraItem = allValue
.filter((i) => i.$original.key === "MODEL_SPECIAL")
.map((obj) => obj.$original.value);
let totalNewLines = 0;
let totalInserts = 0;
let hasMoreLines = true;
//Process file content
if (contentFile.length > 50000) {
for (let i = 0; i < contentFile.length; i += 1000) {
const chunk = contentFile.slice(i, i + 1000);
// ✅ FIX: Process file in batches
while (hasMoreLines) {
// Read batch of lines
const newLines = await readLinesFromPosition(filePath, lastLine, MAX_LINES_PER_BATCH);
chunk.map(async (line, index) => {
//check line the line with errors and exclude errors
listKeyValues
.map((obj) => obj.$original.value)
.map(async (value) => {
if (
line.includes(value) &&
listExcludeErr.filter((err) => line.includes(err))
.length === 0
) {
let log = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
if (newLines.length === 0) {
break; // No more lines
}
let checkLog = allReport?.filter(
(report) =>
report.$original.id_file === log?.id_ldf &&
report.$original.line === index + 1 &&
report.$original.detected_content === value
);
totalNewLines += newLines.length;
if (!log?.id_ldf) {
let logBackup = cacheFile.filter(
(i) => i.file_name === fileName
)[0];
if (logBackup !== undefined) {
let checkLog2 = allReport?.filter(
(report) =>
report.$original.id_file === logBackup?.id_ldf &&
report.$original.line === index + 1 &&
report.$original.detected_content === value
);
if (log && checkLog2?.length === 0) {
await LogReport.create({
detected_content: value,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
} else {
console.log("ERROR CHECK ", fileName);
}
}
if (log && checkLog?.length === 0) {
await LogReport.create({
detected_content: value,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
}
});
// For checkIndexSN - only call once on first batch
if (lastLine === Math.max(0, ...existedReports.map(r => r.line))) {
const contentFileForSN = fs.readFileSync(filePath, "utf8").split("\n");
checkIndexSN(contentFileForSN, lastLine, fileName);
}
if (
checkSpecialVersion(line) !== "" &&
listExcludeErr.filter((err) => line.includes(err)).length ===
0
) {
let checkVersion = checkSpecialVersion(line);
let log = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
if (!log?.id_ldf) {
console.log("ERROR CHECK ", fileName);
}
let checkLog = allReport?.filter(
(report) =>
report.$original.id_file === log?.id_ldf &&
report.$original.line === index + 1 &&
report.$original.detected_content === checkVersion
);
const inserts: any[] = [];
if (log && checkLog?.length === 0) {
await LogReport.create({
detected_content: checkVersion,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
}
});
}
} else {
contentFile.map(async (line, index) => {
//check line the line with errors and exclude errors
listKeyValues
.map((obj) => obj.$original.value)
.map(async (value) => {
if (
line.includes(value) &&
listExcludeErr.filter((err) => line.includes(err))
.length === 0
) {
let log = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
if (!log?.id_ldf) {
console.log("ERROR CHECK ", fileName);
}
let checkLog = allReport?.filter(
(report) =>
report.$original.id_file === log?.id_ldf &&
report.$original.line === index + 1 &&
report.$original.detected_content === value
);
// Process each line in the batch
for (const { lineNo, content: line } of newLines) {
if (!line) continue;
if (EXCLUDE_ERR.some(e => line.includes(e))) continue;
if (log && checkLog?.length === 0) {
await LogReport.create({
detected_content: value,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
}
// if(checkSpecialVersion())
});
if (
checkSpecialVersion(line) !== "" &&
listExcludeErr.filter((err) => line.includes(err)).length === 0
) {
let checkVersion = checkSpecialVersion(line);
let log = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
if (!log?.id_ldf) {
console.log("ERROR CHECK ", fileName);
}
let checkLog = allReport?.filter(
(report) =>
report.$original.id_file === log?.id_ldf &&
report.$original.line === index + 1 &&
report.$original.detected_content === checkVersion
);
if (log && checkLog?.length === 0) {
await LogReport.create({
detected_content: checkVersion,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
}
// ✅ FIX: Use normal loop instead of spread operator
for (const key of keysToCheck) {
if (line.includes(key)) {
const uniq = `${lineNo}-${key}`;
if (!existedSet.has(uniq)) {
existedSet.add(uniq);
inserts.push({
id_file: fileDetect.id_ldf,
line: lineNo,
detected_content: key,
});
}
}
}
//true: import log to log_report table, send report to Zulip
setTimeout(async () => {
if (lines.length === 0) {
console.log(
fileName + "has changed(" + contentFile.length + ") ---Good"
);
} else {
console.log(
fileName +
"has changed(" +
contentFile.length +
") ---SOS---" +
lines.length
);
let allReport_new = await LogReport.query().whereRaw(
`DATE(created_at) = ?`,
[today]
);
const version = checkSpecialVersion(line);
if (version) {
const uniq = `${lineNo}-${version}`;
if (!existedSet.has(uniq)) {
existedSet.add(uniq);
inserts.push({
id_file: fileDetect.id_ldf,
line: lineNo,
detected_content: version,
});
}
}
}
let fileDetect = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
// ✅ FIX: Insert in smaller sub-batches to prevent transaction timeout
if (inserts.length > 0) {
totalInserts += inserts.length;
let logsDetect = allReport_new?.filter(
(i) => i.$original.id_file === fileDetect?.id_ldf
);
for (let i = 0; i < inserts.length; i += MAX_INSERTS_PER_BATCH) {
const subBatch = inserts.slice(i, i + MAX_INSERTS_PER_BATCH);
//Get all report newest
let listReport: any = await getListLineByItem(
logsDetect
.map((obj) => obj.$original)
.filter((l) => l.line > lastLine)
);
let trx;
try {
trx = await Database.transaction();
await LogReport.createMany(subBatch, { client: trx });
await trx.commit();
} catch (error: any) {
if (trx) await trx.rollback();
let content =
"| |Last updated at | Item/error | Line | Report \n|---|:---:|:---|:---|:-----------:\n";
let spoiler = "";
let issueFound = "";
let important = [
"Vxx",
"V00",
"(CAT3K_CAA-UNIVERSALK9-M), Version",
];
listReport.map((log, index) => {
let item = listExtraItem.includes(log.detected_content)
? ":medal: **" + log.detected_content + "**"
: ":small_orange_diamond: " + log.detected_content;
// Check for disk space errors
if (error.code === 'EE_WRITE' || error.errno === 3 ||
(error.sqlMessage && error.sqlMessage.includes('No space left'))) {
console.error("💥 CRITICAL: Disk full! Cannot write to database.");
log.line?.map((line) => {
issueFound =
issueFound +
"\n`" +
line +
"` " +
contentFile[line - 1]?.replace(
log.detected_content,
"[" +
log.detected_content +
"](https://logs.danielvu.com/logs/" +
fileName +
"#" +
line +
")"
);
});
content =
content +
"|" +
(index + 1) +
"|**" +
moment(Date.now()).format("HH:mm - DD/MM") +
"**|" +
item +
"|" +
log.line +
"|[View](https://logs.danielvu.com/logs/" +
fileName +
"#" +
log.line +
")\n";
});
let icon = "";
if (
checkSpecialVersion(content) !== "" ||
important.filter((i) => content.includes(i)).length > 0
) {
icon +=
"------------\n\n:no_entry: :no_entry:**" +
fileName +
"**:no_entry: :no_entry:";
} else {
icon +=
"------------\n\n:warning: :warning: **" + fileName + "**";
}
sendMessToZulip(
try {
await sendMessToZulip(
"stream",
Env.get("ZULIP_STREAM_ALERT"),
Env.get("ZULIP_TOPIC_ALERT"),
icon +
"\n\n" +
content +
"\n\n" +
spoiler +
"\n\n***Issue found:***\n" +
issueFound
`🚨 **CRITICAL DISK FULL ERROR**\n\nFile: ${fileName}\nError: ${error.sqlMessage}\n\n**ACTION REQUIRED: Clean disk space immediately!**`
);
} catch (alertError) {
console.error("Failed to send critical alert:", alertError);
}
}, 3000);
});
watcher.on("error", (error) => {
console.error(`Watcher error: ${error}`);
});
} catch (error) {
console.log(error);
return; // Stop processing
}
console.error("Database transaction error:", error);
throw error;
}
}
// Send notification for this batch
await sendBatchNotification(fileName, inserts, filePath, MODEL_SPECIAL);
}
} catch (error) {
console.log(error);
// Update lastLine for next iteration
lastLine = newLines[newLines.length - 1].lineNo;
// Check if we got fewer lines than max, meaning we've reached the end
if (newLines.length < MAX_LINES_PER_BATCH) {
hasMoreLines = false;
}
}
if (totalInserts === 0) {
console.log(`${fileName} (${totalNewLines} new lines) --- Good`);
} else {
console.log(`${fileName} - Processed ${totalNewLines} lines, found ${totalInserts} issues`);
}
}
/**
* ======================================================
* NOTIFICATION HELPER
* ======================================================
*/
async function sendBatchNotification(
fileName: string,
inserts: any[],
filePath: string,
MODEL_SPECIAL: string[]
) {
try {
// Read file content for building notification
const contentFile = fs.readFileSync(filePath, "utf8").split("\n");
const listReport = await getListLineByItem(inserts);
let table =
"| |Last updated at | Item/error | Line | Report |\n" +
"|---|:---:|:---|:---|:-----------:|\n";
let issueFound = "";
listReport.forEach((log, index) => {
const item = MODEL_SPECIAL.includes(log.detected_content)
? `:medal: **${log.detected_content}**`
: `:small_orange_diamond: ${log.detected_content}`;
log.line.forEach((ln: number) => {
const lineContent = contentFile[ln - 1];
if (lineContent) {
issueFound +=
`\n\`${ln}\` ` +
lineContent.replace(
log.detected_content,
`[${log.detected_content}](https://logs.danielvu.com/logs/${fileName}#${ln})`
);
}
});
table +=
`|${index + 1}|**${moment().format("HH:mm - DD/MM")}**|${item}|${log.line}` +
`|[View](https://logs.danielvu.com/logs/${fileName}#${log.line})|\n`;
});
const icon =
checkSpecialVersion(table) !== ""
? `------------\n\n:no_entry: :no_entry:**${fileName}**:no_entry: :no_entry:`
: `------------\n\n:warning: :warning: **${fileName}**`;
await sendMessToZulip(
"stream",
Env.get("ZULIP_STREAM_ALERT"),
Env.get("ZULIP_TOPIC_ALERT"),
`${icon}\n\n${table}\n\n***Issue found:***\n${issueFound}`
);
} catch (error) {
console.error("Zulip notification error:", error);
// Don't throw - notification failure shouldn't break the process
}
}
/**
* ======================================================
* MAIN ENTRY
* ======================================================
*/
export async function runtimeCheckLogs(folderPath: string) {
const watcher = chokidar.watch(folderPath, {
persistent: true,
ignoreInitial: true,
depth: 0,
awaitWriteFinish: {
stabilityThreshold: 2000,
pollInterval: 100,
},
});
watcher.on("add", async (filePath) => {
if (!filePath.endsWith(".log")) return;
const fileName = filePath.split("/").pop()!;
const today = moment().format("YYYYMMDD");
if (!fileName.startsWith(today)) return;
try {
await LogDetectFile.firstOrCreate(
{ file_name: fileName },
{ file_name: fileName }
);
console.log("New log file:", fileName);
// ⬇️ xử lý luôn file mới
setTimeout(() => {
safeProcessFileChange(filePath);
}, 2000); // đợi file ổn định
} catch (error) {
console.error("Error creating LogDetectFile:", error);
}
});
watcher.on("change", (filePath) => {
if (!filePath.endsWith(".log")) return;
clearTimeout(debounceMap.get(filePath));
debounceMap.set(
filePath,
setTimeout(() => {
safeProcessFileChange(filePath);
}, 1500) // ✅ Increased debounce to reduce load
);
});
watcher.on("error", (err) => {
console.error("Watcher error:", err);
});
console.log("Log watcher started:", folderPath);
}

6
config/database.ts
View File

@ -40,11 +40,11 @@ const databaseConfig: DatabaseConfig = {
port: Env.get("MYSQL_PORT"),
user: Env.get("MYSQL_USER"),
password: Env.get("MYSQL_PASSWORD", ""),
database: Env.get("MYSQL_DB_NAME"),
database: Env.get("MYSQL_DB_NAME")
},
pool: {
min: 1,
max: 2000,
max: 1000,
acquireTimeoutMillis: 60 * 1000
},
@ -52,7 +52,7 @@ const databaseConfig: DatabaseConfig = {
naturalSort: true,
},
healthCheck: false,
healthCheck: true,
debug: false,
},
},

4240
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@ -38,12 +38,15 @@
"dotenv": "^16.3.1",
"form-data": "^4.0.4",
"fs": "^0.0.1-security",
"googleapis": "^169.0.0",
"helpers": "^0.0.6",
"install": "^0.13.0",
"jsonwebtoken": "^9.0.1",
"luxon": "^3.4.0",
"moment": "^2.29.4",
"mysql2": "^3.6.0",
"nodemailer": "^6.9.5",
"npm": "^11.7.0",
"path": "^0.12.7",
"pdfkit": "^0.17.2",
"proxy-addr": "^2.0.7",

9
start/kernel.ts
View File

@ -8,7 +8,6 @@
| preferred, since they keep this file clean.
|
*/
import Server from '@ioc:Adonis/Core/Server'
/*
|--------------------------------------------------------------------------
@ -43,3 +42,11 @@ Server.middleware.registerNamed({
checkToken: () => import('App/Middleware/CheckToken'),
writeLog: () => import('App/Middleware/Logger')
})
process.on("unhandledRejection", (reason) => {
console.error("UNHANDLED REJECTION:", reason);
});
process.on("uncaughtException", (err) => {
console.error("UNCAUGHT EXCEPTION:", err);
});

122
start/routes.ts
View File

@ -37,7 +37,7 @@ const { DocumentProcessorServiceClient } =
require("@google-cloud/documentai").v1;
const fs = require("fs");
const path = require("path");
const { google } = require('googleapis');
runtimeCheckLogs(Env.get("FOLDER_LOGS"));
Route.post("/api/getIndexSerialNumber", "ErpsController.getIndexSerialNumber")
@ -95,6 +95,7 @@ Route.post("/api/backupProduct", async ({ request, response }) => {
PID: res.data[index].PID,
SN: res.data[index].SN,
VID: res.data[index].VID,
brand: res.data[index].brand,
line: res.data[index].line.join(","),
file: res.data[index].fileName,
warehouse: res.data[index].warehouse,
@ -335,11 +336,15 @@ Route.post("/api/test", async () => {
Route.post('/api/wiki/page/insert', async ({ request, response }) => {
try {
const { title, data: dataPayload, pid, vid, sn, lineNumber, license } = request.all()
const { title, data: dataPayload, pid, vid, sn, lineNumber, license, titleAuto, healthChecking } = request.all()
if (!title || !dataPayload) {
return response.status(422).send({ error: `'title' & 'text' is required` })
}
// if (healthChecking) {
// return response.status(200).send({ message: `Checking api wiki success` })
// }
let text = ""
if (typeof dataPayload === "string")
text = dataPayload.replace(/\\n/g, '\n')
@ -363,7 +368,9 @@ License: ${license ?? ""}
)
const loginToken = (await loginTokenRes.json())?.query?.tokens?.logintoken
if (!loginToken) return response.status(500).send('Không lấy được loginToken')
if (healthChecking) {
return response.status(200).send({ message: `Checking api wiki success` })
}
// gửi login
const bodyLogin = new URLSearchParams()
bodyLogin.append('username', 'Ips')
@ -380,8 +387,10 @@ License: ${license ?? ""}
body: bodyLogin,
}
)
const loginCookies = loginRes.headers.raw()['set-cookie']?.join('; ') ?? ''
// const loginCookies = loginRes.headers.raw()['set-cookie']?.join('; ') ?? ''
const loginData = await loginRes.json()
const setCookies = loginRes.headers.getSetCookie()
const loginCookies = setCookies.join('; ')
if (loginData?.clientlogin?.status !== 'PASS') {
return response.status(500).send(loginData)
}
@ -398,45 +407,58 @@ License: ${license ?? ""}
// ============ LẤY PAGE CŨ ============
const pageRes = await fetch(
`https://jv.ipsupply.com.au/wiki/api.php?action=query&prop=revisions&rvprop=content&format=json&titles=${encodeURIComponent(title)}`,
`https://jv.ipsupply.com.au/wiki/api.php?action=query&prop=revisions&rvslots=main&rvprop=content&format=json&titles=${encodeURIComponent(title)}`,
{
headers: { cookie: loginCookies },
method: 'GET',
headers: { cookie: loginCookies }
}
)
const pageJson = await pageRes.json()
// console.log("LOG:", pageJson.query)
const pageId = Object.keys(pageJson.query.pages)[0]
const oldContent = pageJson.query.pages[pageId]?.revisions?.[0]?.['*'] ?? ''
const oldContent = pageJson.query.pages[pageId]?.revisions?.[0]?.slots?.main?.['*'] ?? ''
// nối nội dung mới vào
const now = new Date();
const lineDate = `${String(now.getDate()).padStart(2, "0")}/${String(now.getMonth() + 1).padStart(2, "0")}/${now.getFullYear()}`;
const scopeDate = `-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/>ngay ${lineDate}<br/>-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/><br/>`
const scopeDate = `-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/>${titleAuto ? titleAuto : `ngay ${lineDate}`}<br/>-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/><br/>`
const newText = `${scopeDate}${text}\n\n${checkDateWiki(oldContent) ? oldContent.replace(scopeDate, "") : oldContent}`
// ============ EDIT PAGE ============
const formData = new FormData()
formData.append('text', newText)
formData.append('token', csrfToken)
const body = new URLSearchParams({
action: 'edit',
title,
text: newText,
summary: 'auto update',
token: csrfToken, // để CUỐI
format: 'json',
})
const editRes = await fetch(
`https://jv.ipsupply.com.au/wiki/api.php?action=edit&format=json&title=${encodeURIComponent(title)}`,
'https://jv.ipsupply.com.au/wiki/api.php',
{
method: 'POST',
headers: { cookie: loginCookies },
body: formData,
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
cookie: loginCookies,
},
body,
}
)
const editResult = await editRes.json()
console.log(editResult)
return response.status(200).send(editResult)
} catch (err) {
console.error(err)
return response.status(500).send(err)
console.error(err.message)
return response.status(500).send({ data: err })
}
})
Route.post("/api/po-email/packing-slip", async ({ request, response }) => {
try {
const { data } = request.all()
console.log(data)
const PDFDocument = require('pdfkit')
const getStream = require('stream-buffers')
@ -645,7 +667,7 @@ Route.post("/api/po-email/packing-slip", async ({ request, response }) => {
doc.end()
return stream
}
console.log("PDF done")
const pdfStream = generatePackingSlip(data)
await new Promise(resolve => pdfStream.on('finish', resolve))
@ -667,3 +689,69 @@ Route.post("/api/po-email/packing-slip", async ({ request, response }) => {
})
}
})
Route.get("/api/ram/sheets", async ({ request, response }) => {
try {
const { sheetId } = request.all();
if (!sheetId) {
return response.status(400).send({ msg: "Missing sheetId parameter" });
}
// 1. Cấu hình Auth
const auth = new google.auth.GoogleAuth({
keyFile: '/home/Log_service/app/utils/google.json',
scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'],
});
const client = await auth.getClient();
const googleSheets = google.sheets({ version: 'v4', auth: client });
// 2. Gọi API Google Sheets
const res = await googleSheets.spreadsheets.values.get({
spreadsheetId: sheetId,
range: 'Sheet1!A:Z', // Mở rộng dải ô để tránh thiếu cột
});
const rows = res.data.values;
if (!rows || rows.length === 0) {
return response.status(200).send({ data: [], msg: "Sheet is empty" });
}
// 3. Xử lý dữ liệu
const headers = rows[0].map(h => h.toLowerCase().trim()); // Chuẩn hóa header (xóa khoảng trắng, viết thường)
const dataRows = rows.slice(1);
const ramProfiles = dataRows.map((row) => {
let obj = {};
headers.forEach((header, index) => {
let value = row[index];
// Ép kiểu số cho các trường định lượng
if (['size', 'speed', 'pc'].includes(header)) {
// Xử lý trường hợp giá trị có dấu phẩy hoặc khoảng trắng (vd: "3,200" hoặc "3 200")
const cleanNumber = value ? String(value).replace(/[, ]/g, '') : 0;
obj[header] = Number(cleanNumber) || 0;
} else {
obj[header] = value || ""; // Tránh trả về undefined
}
});
return obj;
});
return response.status(200).send({
data: ramProfiles,
msg: "Get data success"
});
} catch (err) {
console.error('Lỗi API Google Sheets:', err.message);
// Trả về lỗi chi tiết hơn để dễ debug
return response.status(500).send({
msg: "Internal Server Error",
error: err.message
});
}
});