update logic check SN RAM

.gitconfig

@@ -10,3 +10,7 @@
 	directory = /home/Log_service/
 	directory = /home/Log_service/
 	directory = /home/Log_service/
+	directory = /home/Log_service/
+	directory = /home/Log_service/
+	directory = /home/Log_service/
+	directory = /home/Log_service/

app/Controllers/Http/ErpsController.ts

@@ -6,7 +6,7 @@ import LogDetectFile from "App/Models/LogDetectFile";
 const path = require("path");
 
 const BASE_URL = Env.get("BASE_URL_LOG");
-const BASE_URL_AUTO = `${BASE_URL}/AUTOlog/`;
+const BASE_URL_AUTO = `${BASE_URL}/`;
 export default class ErpsController {
   /**
    * Controller do tim cac serial number trong cac log trong khoang thoi gian xac dinh
@@ -78,8 +78,7 @@ export default class ErpsController {
       return null;
     };
 
-    const extractInfoFromLine = (line, index, fName, lines) => {
+    const extractInfoFromLine = (cleanedLine, index, fName, lines, brand, PID) => {
-      const cleanedLine = line.replace(/\r/g, "");
       const results: any = [];
 
       const getWarehouse = (fileName) =>
@@ -98,19 +97,21 @@ export default class ErpsController {
         const SN = extractSN(
           parts.find((p) => p.includes("SN:"))?.split(":")[1] ?? ""
         );
-        const PID = extractSN(
+        PID = extractSN(
           parts.find((p) => p.includes("PID:"))?.split(":")[1] ?? ""
         );
         const VID = extractSN(
           parts.find((p) => p.includes("VID:"))?.split(":")[1] ?? ""
         );
 
+
         if (SN && SN !== "N/A" && SN.length > 4) {
           results.push({
             PID,
             VID,
             SN,
             line: [index + 1],
+            brand,
             fileName: fName,
             warehouse: getWarehouse(fName),
           });
@@ -122,10 +123,11 @@ export default class ErpsController {
         const PCB_SN = extractSN(cleanedLine.split(":")[1]);
         if (PCB_SN) {
           results.push({
-            PID: "",
+            PID,
             VID: "",
             SN: PCB_SN,
             line: [index + 1],
+            brand,
             fileName: fName,
             warehouse: getWarehouse(fName),
           });
@@ -137,10 +139,11 @@ export default class ErpsController {
         const PCB_SN = extractSN(cleanedLine.split(":")[1]);
         if (PCB_SN) {
           results.push({
-            PID: "",
+            PID,
             VID: "",
             SN: PCB_SN.trim(),
             line: [index + 1],
+            brand,
             fileName: fName,
             warehouse: getWarehouse(fName),
           });
@@ -151,10 +154,11 @@ export default class ErpsController {
         const PCB_SN = extractSN(cleanedLine.split(":")[1]);
         if (PCB_SN) {
           results.push({
-            PID: "",
+            PID,
             VID: "",
             SN: PCB_SN,
             line: [index + 1],
+            brand,
             fileName: fName,
             warehouse: getWarehouse(fName),
           });
@@ -166,16 +170,19 @@ export default class ErpsController {
         const PBID = extractSN(cleanedLine.split(" ").pop());
         if (PBID?.length >= 8) {
           results.push({
-            PID: "",
+            PID,
             VID: "",
             SN: PBID,
             line: [index + 1],
+            brand,
             fileName: fName,
             warehouse: getWarehouse(fName),
           });
         }
       }
 
+
+
       return results;
     };
 
@@ -211,9 +218,33 @@ export default class ErpsController {
 
           const fName = path.basename(urls[index] ?? "").trim();
           const output = [];
+          //Vendor Description: --> brand
+          let currentBrand = ""
+          let currentPID = ""
 
           lines.forEach((line, i) => {
-            const lineResult = extractInfoFromLine(line, i, fName, lines);
+            const cleanedLine = line.replace(/\r/g, "");
+            if (cleanedLine.includes("Vendor Description")) {
+              
+              currentBrand = cleanedLine.split("Vendor Description:")[1]?.trim();
+            }
+
+            if (cleanedLine.includes("Vendor Part Number")) {
+              // console.log("Brand: ", cleanedLine)
+              // console.log("Brand1: ", cleanedLine.split("Vendor Part Number:")[1])
+              currentPID = cleanedLine.split("Vendor Part Number:")[1]?.trim();
+            }
+
+            const lineResult = extractInfoFromLine(cleanedLine, i, fName, lines, currentBrand, currentPID);
+            // Nếu dòng này tạo ra SN nào đó -> coi như brand đã được dùng xong
+            if (lineResult.length > 0 && currentBrand) {
+              currentBrand = "";
+            }
+
+            if (lineResult.length > 0 && currentPID) {
+              currentPID = "";
+            }
+
             mergeLines(output, lineResult);
           });
 
@@ -252,8 +283,7 @@ export default class ErpsController {
     const { fileName, line, range } = request.all();
 
     try {
-      let fName =
+      let fName = fileName;
-        fileName.search("AUTO") !== -1 ? "AUTOlog/" + fileName : fileName;
       const res = await axios.get(BASE_URL + "/" + fName);
       const arrayLine = res?.data?.split("\n");
 

app/utils/checkIndexSN.ts

@@ -7,6 +7,7 @@ type OutputItem = {
   PID: string;
   VID: string;
   SN: string;
+  brand: string;
   line: number[];
   fileName: string;
   warehouse: "US" | "AU";
@@ -22,7 +23,8 @@ export const checkIndexSN = async (
     const output: OutputItem[] = [];
 
     if (!arrayLine) return;
-
+    let PID = ""
+    let brand = ""
     for (let i = 0; i < arrayLine.length; i++) {
       const line = arrayLine[i];
       const SN = line
@@ -30,6 +32,21 @@ export const checkIndexSN = async (
         ?.trim()
         .replace(/[!@#$%^&*()_+{}\[\]:;<>,.?~\\/]/g, "");
 
+        if (line.includes("Vendor Part Number")) {
+          PID = line.split("Vendor Part Number:")[1]?.trim();
+        }
+        //Vendor Description: --> brand
+      if (line.includes("Vendor Description") && i >= beginLine) {
+        const rawBrand = line.split("Vendor Description")[1]?.split(":")[1]
+          ?.replace("\r", "")
+          ?.trim()
+          ?.replace(/[!@#$%^&*()_+{}\[\]:;<>,.?~\\/]/g, "");
+
+        if (!rawBrand) continue;
+
+        brand = rawBrand.includes(" ") ? rawBrand.split(" ")[0] : rawBrand;
+        console.log(brand)
+      }
       // Pattern: PID + SN
       if (
         line.includes("PID:") &&
@@ -41,7 +58,7 @@ export const checkIndexSN = async (
         i >= beginLine
       ) {
         const existingIndex = output.findIndex((u) => u.SN === SN);
-        const PID = line?.split("VID:")[0]
+        PID = line?.split("VID:")[0]
           ?.slice(line?.split("VID:")[0]?.search("PID"))
           ?.split(":")[1]
           ?.split(",")[0]
@@ -62,6 +79,7 @@ export const checkIndexSN = async (
             VID,
             SN: SN.includes(" ") ? SN.split(" ")[0] : SN,
             line: [i + 1],
+            brand: brand,
             fileName: nameF,
             warehouse:
               (nameF.includes("-US") ||
@@ -90,10 +108,11 @@ export const checkIndexSN = async (
           existing.line.push(i + 1);
         } else {
           output.push({
-            PID: "",
+            PID,
             VID: "",
             SN,
             line: [i + 1],
+            brand: brand,
             fileName: nameF,
             warehouse:
               (nameF.includes("-US") ||
@@ -122,10 +141,11 @@ export const checkIndexSN = async (
           existing.line.push(i + 1);
         } else {
           output.push({
-            PID: "",
+            PID,
             VID: "",
             SN,
             line: [i + 1],
+            brand: brand,
             fileName: nameF,
             warehouse:
               (nameF.includes("-US") ||
@@ -137,7 +157,6 @@ export const checkIndexSN = async (
           });
         }
       }
-
       //System serial num
       if (line.includes("System serial num") && i >= beginLine) {
         const rawSN = line.split("System serial num")[1]?.split(":")[1]
@@ -154,10 +173,11 @@ export const checkIndexSN = async (
           existing.line.push(i + 1);
         } else {
           output.push({
-            PID: "",
+            PID,
             VID: "",
             SN,
             line: [i + 1],
+            brand: brand,
             fileName: nameF,
             warehouse:
               (nameF.includes("-US") ||
@@ -187,10 +207,11 @@ export const checkIndexSN = async (
           existing.line.push(i + 1);
         } else {
           output.push({
-            PID: "",
+            PID,
             VID: "",
             SN,
             line: [i + 1],
+            brand: brand,
             fileName: nameF,
             warehouse:
               (nameF.includes("-US") ||
@@ -205,9 +226,7 @@ export const checkIndexSN = async (
     }
 
     const pattern = /[\x00-\x20\x7F]/g;
-    const invalidPIDItems = output.filter(
+    const invalidPIDItems = output
-      (i) => !i.PID.match(pattern)
-    );
 
 
     const fileName =

app/utils/google.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "export-items-sold-out",
+  "private_key_id": "0020435b2b32237dc52aaabc1829fecacf57b6be",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCU4x7urMjxRfZ3\nX3BOTaXOh0epkImJuFuYCAEvLFmECqTOJUuTMqTQaknVde53d4H7WNwuT9kqZLK7\nkyVjRSww2H4NFMAt6eWa9K/vjOBJgFrJVdp31IyGBs5IoD47ENV6ZCNk79K2+wRn\n2Vg9jfkHBGgKPVNrbYh4oK6e7wrJ99En3SJvkmOJu1IbVe55/vo24aJQtaxBNwfR\nNWb81EzmQWOnj4iYkbD52WNiWwYhFBdpGr0mAadhxrcERYM/EcLB8dMYJHsA73sm\nXedUv3ocGUuf3U8WDxcAvhkSq7TMk6F+0YcoB1XWZBCbSsBWcR31RQHaVZ3Kchu7\nT51hvmbLAgMBAAECggEAQBeVVUFkLxjjBHg9LDgYzFNH8iKaUoLeSxAHEnuQ/Vpn\nn6ZGZlNCx8U5i3587Uiw17Zk8jrFTY6TslOhoJ5Ca7SGbrMkF5+8i3wR0JlgY8Ox\nUUJ4DWWeKIP9gR4aK+sPOuSop+blWoesWdgiOD1Jmk+3SdEke/Zm7SaehSR6DH7N\nbOKUD2M71FOWlxU5V10J+CAfVgAvC4rl5TD59LijyzVUi46XxvizDBfqPWuskEK0\np35uf/kHKhvc7O8wolGHAECuhQ7btXSyS/A19aTfIquc5PzIyV48Kif+PBr7VAse\nAADf4k0Bx6q+8suA2ftG41jzNYhQCo8osslJanaGgQKBgQDNDIfgzjSjTuq2u8NU\nCCXxRYddVL6zgdeU0COJ23YnXp/6BX2K5vvL3sTU8Oz0coLFuoVNnZ6WKW1hHTAS\nUAxbfl6FtBKh+MsGmT7TWDvkog+Uxr85GNgmwZv2m5Kmm3T1sftYj5kWLWTgs9u4\nMZRfo6B2hSUYF3ELG6zPiQvkWwKBgQC54g6UCscp0hFxSASSIKFz5fkJJe2PtSfn\nAeDoS1bh8meqvbxyJ8NR6nPHDb356IDw5S6gJf28JhB18gILZftWVWZg1IENu0x+\nILeK/HO3F25w+MDAT0D6jSIhjpIcGjHH5tXR8ArndAkJO/I/TSZ9gXJlPC993pE9\n53LtUWJSUQKBgQCE1l5ctBoDxkdmdIOZN+IkyS9ltlYJAs//10t+rs9/t9yZ496u\nEvAC5mHLtf08a8bug+SCIN+xVktHA9q1smgfCeIAamU7y2rz7ml2D45VkuvXLV6K\nhd7aVMlOXQEHsGR4G4LCn9zEMkpNDBgG3w/JWb7FqYIbdmxEjvxIQveSSwKBgGlh\nUOZ43cRFxYG1VHILcbyk6fRXvkA0cXSzSfAnlg/uPvs7KbSF+mNcSVrV3ZfqSf9H\nzpEYOKsL4VUcaDBSFpUxQSMwFiHUx17R6Xwv9uIbtI7zV0MvKyVgEbN4gjjQWNx9\nb0bfkS76R40moQVmFaYdVt8xqbLwFONFbZBBDT6hAoGBALdoPSCWDQNkWYrHm5nN\nFCqbF8JjDoolLh+KkiA9lphT5wxJMf35JIhUodEH1WFoLHNjSP1jCNk0UVj/4/aS\nrXphqiCmZNriqfec7B9eWsR7h482rl24DaOHVYjeBR6uqGUVJKVuQ/KkpKC15Rbm\nRYcVX/gCpP9/QNYPsPnBriEG\n-----END PRIVATE KEY-----\n",
+  "client_email": "export-file-service@export-items-sold-out.iam.gserviceaccount.com",
+  "client_id": "111585688432424787777",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/export-file-service%40export-items-sold-out.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

app/utils/runtimeCheckLogs.ts

@@ -1,9 +1,11 @@
 import Env from "@ioc:Adonis/Core/Env";
+import Database from "@ioc:Adonis/Lucid/Database";
 import KeyValue from "App/Models/KeyValue";
 import LogDetectFile from "App/Models/LogDetectFile";
 import LogReport from "App/Models/LogReport";
 import chokidar from "chokidar";
 import fs from "fs";
+import readline from "readline";
 import { DateTime } from "luxon";
 import moment from "moment";
 import { checkIndexSN } from "./checkIndexSN";
@@ -11,423 +13,371 @@ import { getListLineByItem } from "./getListLineByItem";
 import { sendMessToZulip } from "./sendMessToZulip";
 import { checkSpecialVersion } from "./helper";
 
-export async function runtimeCheckLogs(folderPath: string) {
+/**
+ * ======================================================
+ * CONSTANTS
+ * ======================================================
+ */
+const MAX_LINES_PER_BATCH = 5000; // Process max 5k lines at a time
+const MAX_INSERTS_PER_BATCH = 500; // Insert max 500 records per transaction
+
+/**
+ * ======================================================
+ * KEY VALUE CACHE (GLOBAL)
+ * ======================================================
+ */
+let keyValueCache: Record<string, string[]> | null = null;
+let keyValueCacheAt = 0;
+const CACHE_TTL = 60_000;
+
+async function getKeyValueCached() {
+  if (keyValueCache && Date.now() - keyValueCacheAt < CACHE_TTL) {
+    return keyValueCache;
+  }
+
+  const rows = await KeyValue.query().select("key", "value");
+  const map: Record<string, string[]> = {};
+
+  for (const r of rows) {
+    if (!map[r.key]) map[r.key] = [];
+    map[r.key].push(r.value);
+  }
+
+  keyValueCache = map;
+  keyValueCacheAt = Date.now();
+  return map;
+}
+
+/**
+ * ======================================================
+ * FILE LOCK + DEBOUNCE
+ * ======================================================
+ */
+const fileLocks = new Set<string>();
+const debounceMap = new Map<string, NodeJS.Timeout>();
+
+async function safeProcessFileChange(filePath: string) {
+  if (fileLocks.has(filePath)) return;
+
+  fileLocks.add(filePath);
   try {
-    let cacheFile: any = [];
+    await processFileChange(filePath);
-    let fileList = Array();
+  } catch (err) {
-    let fileList_old = Array();
+    console.error("processFileChange error:", err);
-    // Function to update the list of files
+  } finally {
-    async function updateFileList() {
+    fileLocks.delete(filePath);
-      //get date now
+  }
-      let dateNow = moment(Date.now()).format("YYYY/MM/DD");
+}
-      const dateCheck = moment(Date.now()).format("YYYYMMDD");
-      //Get list file in folder
-      fileList = fs.readdirSync(folderPath);
-      //Get date modified
-      fileList.map((file) => {
-        const filePath = `${folderPath}/${file}`;
-        if (file?.split(".")[filePath.split(".").length - 1] === "log") {
-          const stats = fs.statSync(filePath);
 
-          //scan file today
+/**
-          if (dateNow !== moment(stats.mtime).format("YYYY/MM/DD")) {
+ * ======================================================
-            fileList = fileList.filter((i) => i !== file && i.includes(dateCheck));
+ * STREAM-BASED FILE READER (MEMORY EFFICIENT)
+ * ======================================================
+ */
+async function readLinesFromPosition(
+  filePath: string,
+  startLine: number,
+  maxLines: number = MAX_LINES_PER_BATCH
+): Promise<Array<{ lineNo: number; content: string }>> {
+  return new Promise((resolve, reject) => {
+    const lines: Array<{ lineNo: number; content: string }> = [];
+    let currentLine = 0;
+
+    const stream = fs.createReadStream(filePath, { encoding: "utf8" });
+    const rl = readline.createInterface({
+      input: stream,
+      crlfDelay: Infinity,
+    });
+
+    rl.on("line", (line) => {
+      currentLine++;
+      if (currentLine > startLine) {
+        lines.push({ lineNo: currentLine, content: line });
+
+        // ✅ FIX: Stop reading after maxLines to prevent memory overflow
+        if (lines.length >= maxLines) {
+          rl.close();
+          stream.destroy();
         }
       }
     });
-      console.log(fileList)
+
+    rl.on("close", () => resolve(lines));
+    rl.on("error", reject);
+    stream.on("error", reject);
+  });
 }
 
-    // Watch the folder for new files
+/**
-    const folderWatcher = chokidar.watch(folderPath);
+ * ======================================================
+ * CORE LOGIC (FIXED FOR STACK OVERFLOW)
+ * ======================================================
+ */
+async function processFileChange(filePath: string) {
+  const fileName = filePath.split("/").pop()!;
 
-    folderWatcher.on("add", async (filePath) => {
+  const fileDetect = await LogDetectFile
-      //import log new file
+    .query()
-      //check format file name
+    .where("file_name", fileName)
-      if (
+    .first();
-        filePath?.split(".")[filePath.split(".").length - 1] === "log" &&
+
-        filePath.split("/")[filePath.split("/").length - 1]?.split("-")[0] ===
+  if (!fileDetect) return;
-          //localhost
+
-          // filePath.split("\\")[filePath.split("\\").length - 1]?.split("-")[0] ===
+  const existedReports = await LogReport
-          moment(Date.now()).format("YYYYMMDD").toString()
+    .query()
-      ) {
+    .select("line", "detected_content")
-        //add information file to database
+    .where("id_file", fileDetect.id_ldf);
-        console.log("New file added: ", filePath);
+
-        await LogDetectFile.firstOrCreate(
+  const existedSet = new Set(
-          { file_name: filePath.split("/")[filePath.split("/").length - 1] },
+    existedReports.map(r => `${r.line}-${r.detected_content}`)
-          { file_name: filePath.split("/")[filePath.split("/").length - 1] }
   );
 
-        //localhost
+  let lastLine = Math.max(0, ...existedReports.map(r => r.line));
-        // await LogDetectFile.firstOrCreate(
-        //   { file_name: filePath.split("\\")[filePath.split("\\").length - 1] },
-        //   { file_name: filePath.split("\\")[filePath.split("\\").length - 1] }
-        // );
 
-        fileList_old = fileList;
+  const keyMap = await getKeyValueCached();
-        updateFileList();
+  const MODEL_SPECIAL = keyMap.MODEL_SPECIAL || [];
-        watchFilesInList();
+  const CATCH_FAULTY = keyMap.CATCH_FAULTY || [];
+  const EXCLUDE_ERR = keyMap.EXCLUDE_ERR || [];
+
+  // ✅ FIX: Combine arrays safely (avoid spread operator with large arrays)
+  const keysToCheck: string[] = [];
+  for (const key of MODEL_SPECIAL) keysToCheck.push(key);
+  for (const key of CATCH_FAULTY) keysToCheck.push(key);
+
+  let totalNewLines = 0;
+  let totalInserts = 0;
+  let hasMoreLines = true;
+
+  // ✅ FIX: Process file in batches
+  while (hasMoreLines) {
+    // Read batch of lines
+    const newLines = await readLinesFromPosition(filePath, lastLine, MAX_LINES_PER_BATCH);
+
+    if (newLines.length === 0) {
+      break; // No more lines
     }
-      // updateFile;
-    });
 
-    // Watch for changes in the files listed
+    totalNewLines += newLines.length;
-    async function watchFilesInList() {
+
+    // For checkIndexSN - only call once on first batch
+    if (lastLine === Math.max(0, ...existedReports.map(r => r.line))) {
+      const contentFileForSN = fs.readFileSync(filePath, "utf8").split("\n");
+      checkIndexSN(contentFileForSN, lastLine, fileName);
+    }
+
+    const inserts: any[] = [];
+
+    // Process each line in the batch
+    for (const { lineNo, content: line } of newLines) {
+      if (!line) continue;
+      if (EXCLUDE_ERR.some(e => line.includes(e))) continue;
+
+      // ✅ FIX: Use normal loop instead of spread operator
+      for (const key of keysToCheck) {
+        if (line.includes(key)) {
+          const uniq = `${lineNo}-${key}`;
+          if (!existedSet.has(uniq)) {
+            existedSet.add(uniq);
+            inserts.push({
+              id_file: fileDetect.id_ldf,
+              line: lineNo,
+              detected_content: key,
+            });
+          }
+        }
+      }
+
+      const version = checkSpecialVersion(line);
+      if (version) {
+        const uniq = `${lineNo}-${version}`;
+        if (!existedSet.has(uniq)) {
+          existedSet.add(uniq);
+          inserts.push({
+            id_file: fileDetect.id_ldf,
+            line: lineNo,
+            detected_content: version,
+          });
+        }
+      }
+    }
+
+    // ✅ FIX: Insert in smaller sub-batches to prevent transaction timeout
+    if (inserts.length > 0) {
+      totalInserts += inserts.length;
+
+      for (let i = 0; i < inserts.length; i += MAX_INSERTS_PER_BATCH) {
+        const subBatch = inserts.slice(i, i + MAX_INSERTS_PER_BATCH);
+
+        let trx;
         try {
-        //only check new file ---> fileList - fileList_old = new file
+          trx = await Database.transaction();
-        let listFileWatch = fileList
+          await LogReport.createMany(subBatch, { client: trx });
-          ?.filter(
+          await trx.commit();
-            (i) => fileList_old.includes(i) === false && i.includes(".log")
+        } catch (error: any) {
-          )
+          if (trx) await trx.rollback();
-          .map((file) => folderPath + "/" + file);
-        const watcher = chokidar.watch(listFileWatch, {
-          persistent: true,
-          usePolling: true,
-          interval: 300000,
-        });
 
-        watcher.setMaxListeners(200);
+          // Check for disk space errors
+          if (error.code === 'EE_WRITE' || error.errno === 3 ||
+            (error.sqlMessage && error.sqlMessage.includes('No space left'))) {
+            console.error("💥 CRITICAL: Disk full! Cannot write to database.");
 
-        watcher.on("change", async (path) => {
+            try {
-          // fs.watchFile(filePath,{ interval: 15000 },
+              await sendMessToZulip(
-          // async (eventType) => {
-          //check special item, extra RAM, error in log
-          const fileName = path.split("/")[path.split("/").length - 1];
-          // const fileName = path.split("\\")[path.split("\\").length - 1];
-          const filePath = path;
-          let lines: any = [];
-          const today = DateTime.now().toFormat("yyyy-MM-dd");
-          let allFile = await LogDetectFile.query().whereRaw(
-            `DATE(created_at) = ?`,
-            [today]
-          );
-          // let allReport = await LogReport.all();
-          let allValue = await KeyValue.all();
-          const allReport = await LogReport.query().whereRaw(
-            `DATE(created_at) = ?`,
-            [today]
-          );
-
-          cacheFile = cacheFile.concat(
-            allFile.filter(
-              (obj2) => !cacheFile.some((obj1) => obj1.id_ldf === obj2.id_ldf)
-            )
-          );
-
-          //get information file
-          let fileDetect = allFile?.filter(
-            (i) => i.$original.file_name === fileName
-          )[0];
-
-          let logsDetect = allReport?.filter(
-            (i) => i.$original.id_file === fileDetect?.id_ldf
-          );
-          //get the last line detected
-          let lastLine = Math.max(...logsDetect.map((obj) => obj.line));
-
-          //get content file in local
-          let contentFile = await fs
-            .readFileSync(filePath)
-            .toString()
-            ?.split("\n");
-
-          //get index SN and send to ERP
-
-          checkIndexSN(contentFile, lastLine, fileName);
-
-          //get list item to check
-          let listKeyValues = allValue.filter(
-            (i) =>
-              i.$original.key === "MODEL_SPECIAL" ||
-              i.$original.key === "CATCH_FAULTY"
-          );
-
-          //get list exclude error
-          let listExcludeErr = allValue
-            .filter((i) => i.$original.key === "EXCLUDE_ERR")
-            .map((obj) => obj.$original.value);
-
-          //get list item special
-          let listExtraItem = allValue
-            .filter((i) => i.$original.key === "MODEL_SPECIAL")
-            .map((obj) => obj.$original.value);
-
-          //Process file content
-          if (contentFile.length > 50000) {
-            for (let i = 0; i < contentFile.length; i += 1000) {
-              const chunk = contentFile.slice(i, i + 1000);
-
-              chunk.map(async (line, index) => {
-                //check line the line with errors and exclude errors
-                listKeyValues
-                  .map((obj) => obj.$original.value)
-                  .map(async (value) => {
-                    if (
-                      line.includes(value) &&
-                      listExcludeErr.filter((err) => line.includes(err))
-                        .length === 0
-                    ) {
-                      let log = allFile?.filter(
-                        (i) => i.$original.file_name === fileName
-                      )[0];
-
-                      let checkLog = allReport?.filter(
-                        (report) =>
-                          report.$original.id_file === log?.id_ldf &&
-                          report.$original.line === index + 1 &&
-                          report.$original.detected_content === value
-                      );
-
-                      if (!log?.id_ldf) {
-                        let logBackup = cacheFile.filter(
-                          (i) => i.file_name === fileName
-                        )[0];
-                        if (logBackup !== undefined) {
-                          let checkLog2 = allReport?.filter(
-                            (report) =>
-                              report.$original.id_file === logBackup?.id_ldf &&
-                              report.$original.line === index + 1 &&
-                              report.$original.detected_content === value
-                          );
-                          if (log && checkLog2?.length === 0) {
-                            await LogReport.create({
-                              detected_content: value,
-                              line: index + 1,
-                              id_file: log?.id_ldf,
-                            });
-                            lines.push(index + 1);
-                          }
-                        } else {
-                          console.log("ERROR CHECK ", fileName);
-                        }
-                      }
-                      if (log && checkLog?.length === 0) {
-                        await LogReport.create({
-                          detected_content: value,
-                          line: index + 1,
-                          id_file: log?.id_ldf,
-                        });
-                        lines.push(index + 1);
-                      }
-                    }
-                  });
-
-                if (
-                  checkSpecialVersion(line) !== "" &&
-                  listExcludeErr.filter((err) => line.includes(err)).length ===
-                    0
-                ) {
-                  let checkVersion = checkSpecialVersion(line);
-                  let log = allFile?.filter(
-                    (i) => i.$original.file_name === fileName
-                  )[0];
-                  if (!log?.id_ldf) {
-                    console.log("ERROR CHECK ", fileName);
-                  }
-                  let checkLog = allReport?.filter(
-                    (report) =>
-                      report.$original.id_file === log?.id_ldf &&
-                      report.$original.line === index + 1 &&
-                      report.$original.detected_content === checkVersion
-                  );
-
-                  if (log && checkLog?.length === 0) {
-                    await LogReport.create({
-                      detected_content: checkVersion,
-                      line: index + 1,
-                      id_file: log?.id_ldf,
-                    });
-                    lines.push(index + 1);
-                  }
-                }
-              });
-            }
-          } else {
-            contentFile.map(async (line, index) => {
-              //check line the line with errors and exclude errors
-              listKeyValues
-                .map((obj) => obj.$original.value)
-                .map(async (value) => {
-                  if (
-                    line.includes(value) &&
-                    listExcludeErr.filter((err) => line.includes(err))
-                      .length === 0
-                  ) {
-                    let log = allFile?.filter(
-                      (i) => i.$original.file_name === fileName
-                    )[0];
-                    if (!log?.id_ldf) {
-                      console.log("ERROR CHECK ", fileName);
-                    }
-                    let checkLog = allReport?.filter(
-                      (report) =>
-                        report.$original.id_file === log?.id_ldf &&
-                        report.$original.line === index + 1 &&
-                        report.$original.detected_content === value
-                    );
-
-                    if (log && checkLog?.length === 0) {
-                      await LogReport.create({
-                        detected_content: value,
-                        line: index + 1,
-                        id_file: log?.id_ldf,
-                      });
-                      lines.push(index + 1);
-                    }
-                  }
-
-                  // if(checkSpecialVersion())
-                });
-
-              if (
-                checkSpecialVersion(line) !== "" &&
-                listExcludeErr.filter((err) => line.includes(err)).length === 0
-              ) {
-                let checkVersion = checkSpecialVersion(line);
-                let log = allFile?.filter(
-                  (i) => i.$original.file_name === fileName
-                )[0];
-                if (!log?.id_ldf) {
-                  console.log("ERROR CHECK ", fileName);
-                }
-                let checkLog = allReport?.filter(
-                  (report) =>
-                    report.$original.id_file === log?.id_ldf &&
-                    report.$original.line === index + 1 &&
-                    report.$original.detected_content === checkVersion
-                );
-
-                if (log && checkLog?.length === 0) {
-                  await LogReport.create({
-                    detected_content: checkVersion,
-                    line: index + 1,
-                    id_file: log?.id_ldf,
-                  });
-                  lines.push(index + 1);
-                }
-              }
-            });
-          }
-
-          //true: import log to log_report table, send report to Zulip
-          setTimeout(async () => {
-            if (lines.length === 0) {
-              console.log(
-                fileName + "has changed(" + contentFile.length + ") ---Good"
-              );
-            } else {
-              console.log(
-                fileName +
-                  "has changed(" +
-                  contentFile.length +
-                  ") ---SOS---" +
-                  lines.length
-              );
-              let allReport_new = await LogReport.query().whereRaw(
-                `DATE(created_at) = ?`,
-                [today]
-              );
-
-              let fileDetect = allFile?.filter(
-                (i) => i.$original.file_name === fileName
-              )[0];
-
-              let logsDetect = allReport_new?.filter(
-                (i) => i.$original.id_file === fileDetect?.id_ldf
-              );
-
-              //Get all report newest
-              let listReport: any = await getListLineByItem(
-                logsDetect
-                  .map((obj) => obj.$original)
-                  .filter((l) => l.line > lastLine)
-              );
-
-              let content =
-                "|  |Last updated at | Item/error | Line | Report \n|---|:---:|:---|:---|:-----------:\n";
-              let spoiler = "";
-              let issueFound = "";
-              let important = [
-                "Vxx",
-                "V00",
-                "(CAT3K_CAA-UNIVERSALK9-M), Version",
-              ];
-              listReport.map((log, index) => {
-                let item = listExtraItem.includes(log.detected_content)
-                  ? ":medal: **" + log.detected_content + "**"
-                  : ":small_orange_diamond: " + log.detected_content;
-
-                log.line?.map((line) => {
-                  issueFound =
-                    issueFound +
-                    "\n`" +
-                    line +
-                    "` " +
-                    contentFile[line - 1]?.replace(
-                      log.detected_content,
-                      "[" +
-                        log.detected_content +
-                        "](https://logs.danielvu.com/logs/" +
-                        fileName +
-                        "#" +
-                        line +
-                        ")"
-                    );
-                });
-                content =
-                  content +
-                  "|" +
-                  (index + 1) +
-                  "|**" +
-                  moment(Date.now()).format("HH:mm - DD/MM") +
-                  "**|" +
-                  item +
-                  "|" +
-                  log.line +
-                  "|[View](https://logs.danielvu.com/logs/" +
-                  fileName +
-                  "#" +
-                  log.line +
-                  ")\n";
-              });
-
-              let icon = "";
-              if (
-                checkSpecialVersion(content) !== "" ||
-                important.filter((i) => content.includes(i)).length > 0
-              ) {
-                icon +=
-                  "------------\n\n:no_entry: :no_entry:**" +
-                  fileName +
-                  "**:no_entry: :no_entry:";
-              } else {
-                icon +=
-                  "------------\n\n:warning: :warning: **" + fileName + "**";
-              }
-              sendMessToZulip(
                 "stream",
                 Env.get("ZULIP_STREAM_ALERT"),
                 Env.get("ZULIP_TOPIC_ALERT"),
-                icon +
+                `🚨 **CRITICAL DISK FULL ERROR**\n\nFile: ${fileName}\nError: ${error.sqlMessage}\n\n**ACTION REQUIRED: Clean disk space immediately!**`
-                  "\n\n" +
+              );
-                  content +
+            } catch (alertError) {
-                  "\n\n" +
+              console.error("Failed to send critical alert:", alertError);
-                  spoiler +
+            }
-                  "\n\n***Issue found:***\n" +
+
-                  issueFound
+            return; // Stop processing
+          }
+
+          console.error("Database transaction error:", error);
+          throw error;
+        }
+      }
+
+      // Send notification for this batch
+      await sendBatchNotification(fileName, inserts, filePath, MODEL_SPECIAL);
+    }
+
+    // Update lastLine for next iteration
+    lastLine = newLines[newLines.length - 1].lineNo;
+
+    // Check if we got fewer lines than max, meaning we've reached the end
+    if (newLines.length < MAX_LINES_PER_BATCH) {
+      hasMoreLines = false;
+    }
+  }
+
+  if (totalInserts === 0) {
+    console.log(`${fileName} (${totalNewLines} new lines) --- Good`);
+  } else {
+    console.log(`${fileName} - Processed ${totalNewLines} lines, found ${totalInserts} issues`);
+  }
+}
+
+/**
+ * ======================================================
+ * NOTIFICATION HELPER
+ * ======================================================
+ */
+async function sendBatchNotification(
+  fileName: string,
+  inserts: any[],
+  filePath: string,
+  MODEL_SPECIAL: string[]
+) {
+  try {
+    // Read file content for building notification
+    const contentFile = fs.readFileSync(filePath, "utf8").split("\n");
+    const listReport = await getListLineByItem(inserts);
+
+    let table =
+      "|  |Last updated at | Item/error | Line | Report |\n" +
+      "|---|:---:|:---|:---|:-----------:|\n";
+
+    let issueFound = "";
+
+    listReport.forEach((log, index) => {
+      const item = MODEL_SPECIAL.includes(log.detected_content)
+        ? `:medal: **${log.detected_content}**`
+        : `:small_orange_diamond: ${log.detected_content}`;
+
+      log.line.forEach((ln: number) => {
+        const lineContent = contentFile[ln - 1];
+        if (lineContent) {
+          issueFound +=
+            `\n\`${ln}\` ` +
+            lineContent.replace(
+              log.detected_content,
+              `[${log.detected_content}](https://logs.danielvu.com/logs/${fileName}#${ln})`
             );
         }
-          }, 3000);
       });
 
-        watcher.on("error", (error) => {
+      table +=
-          console.error(`Watcher error: ${error}`);
+        `|${index + 1}|**${moment().format("HH:mm - DD/MM")}**|${item}|${log.line}` +
+        `|[View](https://logs.danielvu.com/logs/${fileName}#${log.line})|\n`;
     });
+
+    const icon =
+      checkSpecialVersion(table) !== ""
+        ? `------------\n\n:no_entry: :no_entry:**${fileName}**:no_entry: :no_entry:`
+        : `------------\n\n:warning: :warning: **${fileName}**`;
+
+    await sendMessToZulip(
+      "stream",
+      Env.get("ZULIP_STREAM_ALERT"),
+      Env.get("ZULIP_TOPIC_ALERT"),
+      `${icon}\n\n${table}\n\n***Issue found:***\n${issueFound}`
+    );
   } catch (error) {
-        console.log(error);
+    console.error("Zulip notification error:", error);
+    // Don't throw - notification failure shouldn't break the process
   }
 }
+
+/**
+ * ======================================================
+ * MAIN ENTRY
+ * ======================================================
+ */
+export async function runtimeCheckLogs(folderPath: string) {
+  const watcher = chokidar.watch(folderPath, {
+    persistent: true,
+    ignoreInitial: true,
+    depth: 0,
+    awaitWriteFinish: {
+      stabilityThreshold: 2000,
+      pollInterval: 100,
+    },
+  });
+
+  watcher.on("add", async (filePath) => {
+    if (!filePath.endsWith(".log")) return;
+
+    const fileName = filePath.split("/").pop()!;
+    const today = moment().format("YYYYMMDD");
+
+    if (!fileName.startsWith(today)) return;
+
+    try {
+      await LogDetectFile.firstOrCreate(
+        { file_name: fileName },
+        { file_name: fileName }
+      );
+
+      console.log("New log file:", fileName);
+
+      // ⬇️ xử lý luôn file mới
+      setTimeout(() => {
+        safeProcessFileChange(filePath);
+      }, 2000); // đợi file ổn định
     } catch (error) {
-    console.log(error);
+      console.error("Error creating LogDetectFile:", error);
     }
+  });
+
+  watcher.on("change", (filePath) => {
+    if (!filePath.endsWith(".log")) return;
+
+    clearTimeout(debounceMap.get(filePath));
+
+    debounceMap.set(
+      filePath,
+      setTimeout(() => {
+        safeProcessFileChange(filePath);
+      }, 1500) // ✅ Increased debounce to reduce load
+    );
+  });
+
+  watcher.on("error", (err) => {
+    console.error("Watcher error:", err);
+  });
+
+  console.log("Log watcher started:", folderPath);
 }

config/database.ts

@@ -40,11 +40,11 @@ const databaseConfig: DatabaseConfig = {
         port: Env.get("MYSQL_PORT"),
         user: Env.get("MYSQL_USER"),
         password: Env.get("MYSQL_PASSWORD", ""),
-        database: Env.get("MYSQL_DB_NAME"),
+        database: Env.get("MYSQL_DB_NAME")
       },
       pool: {
         min: 1,
-        max: 2000,
+        max: 1000,
         acquireTimeoutMillis: 60 * 1000
       },
 
@@ -52,7 +52,7 @@ const databaseConfig: DatabaseConfig = {
         naturalSort: true,
       },
       
-      healthCheck: false,
+      healthCheck: true,
       debug: false,
     },
   },

package.json

@@ -38,12 +38,15 @@
     "dotenv": "^16.3.1",
     "form-data": "^4.0.4",
     "fs": "^0.0.1-security",
+    "googleapis": "^169.0.0",
     "helpers": "^0.0.6",
+    "install": "^0.13.0",
     "jsonwebtoken": "^9.0.1",
     "luxon": "^3.4.0",
     "moment": "^2.29.4",
     "mysql2": "^3.6.0",
     "nodemailer": "^6.9.5",
+    "npm": "^11.7.0",
     "path": "^0.12.7",
     "pdfkit": "^0.17.2",
     "proxy-addr": "^2.0.7",

start/kernel.ts

@@ -8,7 +8,6 @@
 | preferred, since they keep this file clean.
 |
 */
-
 import Server from '@ioc:Adonis/Core/Server'
 /*
 |--------------------------------------------------------------------------
@@ -43,3 +42,11 @@ Server.middleware.registerNamed({
   checkToken: () => import('App/Middleware/CheckToken'),
   writeLog: () => import('App/Middleware/Logger')
 })
+
+process.on("unhandledRejection", (reason) => {
+  console.error("UNHANDLED REJECTION:", reason);
+});
+
+process.on("uncaughtException", (err) => {
+  console.error("UNCAUGHT EXCEPTION:", err);
+});

start/routes.ts

@@ -37,7 +37,7 @@ const { DocumentProcessorServiceClient } =
   require("@google-cloud/documentai").v1;
 const fs = require("fs");
 const path = require("path");
-
+const { google } = require('googleapis');
 runtimeCheckLogs(Env.get("FOLDER_LOGS"));
 
 Route.post("/api/getIndexSerialNumber", "ErpsController.getIndexSerialNumber")
@@ -95,6 +95,7 @@ Route.post("/api/backupProduct", async ({ request, response }) => {
         PID: res.data[index].PID,
         SN: res.data[index].SN,
         VID: res.data[index].VID,
+        brand: res.data[index].brand,
         line: res.data[index].line.join(","),
         file: res.data[index].fileName,
         warehouse: res.data[index].warehouse,
@@ -335,11 +336,15 @@ Route.post("/api/test", async () => {
 
 Route.post('/api/wiki/page/insert', async ({ request, response }) => {
   try {
-    const { title, data: dataPayload, pid, vid, sn, lineNumber, license } = request.all()
+    const { title, data: dataPayload, pid, vid, sn, lineNumber, license, titleAuto, healthChecking } = request.all()
     if (!title || !dataPayload) {
       return response.status(422).send({ error: `'title' & 'text' is required` })
     }
 
+    // if (healthChecking) {
+    //   return response.status(200).send({ message: `Checking api wiki success` })
+    // }
+
     let text = ""
     if (typeof dataPayload === "string")
       text = dataPayload.replace(/\\n/g, '\n')
@@ -363,7 +368,9 @@ License:  ${license ?? ""}
     )
     const loginToken = (await loginTokenRes.json())?.query?.tokens?.logintoken
     if (!loginToken) return response.status(500).send('Không lấy được loginToken')
-
+    if (healthChecking) {
+      return response.status(200).send({ message: `Checking api wiki success` })
+    }
     // gửi login
     const bodyLogin = new URLSearchParams()
     bodyLogin.append('username', 'Ips')
@@ -380,8 +387,10 @@ License:  ${license ?? ""}
         body: bodyLogin,
       }
     )
-    const loginCookies = loginRes.headers.raw()['set-cookie']?.join('; ') ?? ''
+    // const loginCookies = loginRes.headers.raw()['set-cookie']?.join('; ') ?? ''
     const loginData = await loginRes.json()
+    const setCookies = loginRes.headers.getSetCookie()
+    const loginCookies = setCookies.join('; ')
     if (loginData?.clientlogin?.status !== 'PASS') {
       return response.status(500).send(loginData)
     }
@@ -398,45 +407,58 @@ License:  ${license ?? ""}
       
     // ============ LẤY PAGE CŨ ============
     const pageRes = await fetch(
-      `https://jv.ipsupply.com.au/wiki/api.php?action=query&prop=revisions&rvprop=content&format=json&titles=${encodeURIComponent(title)}`,
+      `https://jv.ipsupply.com.au/wiki/api.php?action=query&prop=revisions&rvslots=main&rvprop=content&format=json&titles=${encodeURIComponent(title)}`,
       {
-        headers: { cookie: loginCookies },
+        method: 'GET',
+        headers: { cookie: loginCookies }
       }
     )
     const pageJson = await pageRes.json()
+    // console.log("LOG:", pageJson.query)
     const pageId = Object.keys(pageJson.query.pages)[0]
-    const oldContent = pageJson.query.pages[pageId]?.revisions?.[0]?.['*'] ?? ''
+    const oldContent = pageJson.query.pages[pageId]?.revisions?.[0]?.slots?.main?.['*'] ?? ''
 
     // nối nội dung mới vào
     const now = new Date();
     const lineDate = `${String(now.getDate()).padStart(2, "0")}/${String(now.getMonth() + 1).padStart(2, "0")}/${now.getFullYear()}`;
-    const scopeDate = `-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/>ngay ${lineDate}<br/>-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/><br/>`
+    const scopeDate = `-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/>${titleAuto ? titleAuto : `ngay ${lineDate}`}<br/>-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*<br/><br/>`
     const newText = `${scopeDate}${text}\n\n${checkDateWiki(oldContent) ? oldContent.replace(scopeDate, "") : oldContent}`
 
     // ============ EDIT PAGE ============
-    const formData = new FormData()
+    const body = new URLSearchParams({
-    formData.append('text', newText)
+      action: 'edit',
-    formData.append('token', csrfToken)
+      title,
+      text: newText,
+      summary: 'auto update',
+      token: csrfToken, // để CUỐI
+      format: 'json',
+    })
     
     const editRes = await fetch(
-      `https://jv.ipsupply.com.au/wiki/api.php?action=edit&format=json&title=${encodeURIComponent(title)}`,
+      'https://jv.ipsupply.com.au/wiki/api.php',
       {
         method: 'POST',
-        headers: { cookie: loginCookies },
+        headers: {
-        body: formData,
+          'Content-Type': 'application/x-www-form-urlencoded',
+          cookie: loginCookies,
+        },
+        body,
       }
     )
+
     const editResult = await editRes.json()
+    console.log(editResult)
     return response.status(200).send(editResult)
   } catch (err) {
-    console.error(err)
+    console.error(err.message)
-    return response.status(500).send(err)
+    return response.status(500).send({ data: err })
   }
 })
 
 Route.post("/api/po-email/packing-slip", async ({ request, response }) => {
   try {
     const { data } = request.all()
+    console.log(data)
     const PDFDocument = require('pdfkit')
     const getStream = require('stream-buffers')
 
@@ -645,7 +667,7 @@ Route.post("/api/po-email/packing-slip", async ({ request, response }) => {
       doc.end()
       return stream
     }
-
+    console.log("PDF done")
     const pdfStream = generatePackingSlip(data)
 
     await new Promise(resolve => pdfStream.on('finish', resolve))
@@ -667,3 +689,69 @@ Route.post("/api/po-email/packing-slip", async ({ request, response }) => {
     })
   }
 })
+
+Route.get("/api/ram/sheets", async ({ request, response }) => {
+  try {
+    const { sheetId } = request.all();
+
+    if (!sheetId) {
+      return response.status(400).send({ msg: "Missing sheetId parameter" });
+    }
+
+    // 1. Cấu hình Auth
+    const auth = new google.auth.GoogleAuth({
+      keyFile: '/home/Log_service/app/utils/google.json',
+      scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'],
+    });
+
+    const client = await auth.getClient();
+    const googleSheets = google.sheets({ version: 'v4', auth: client });
+
+    // 2. Gọi API Google Sheets
+    const res = await googleSheets.spreadsheets.values.get({
+      spreadsheetId: sheetId,
+      range: 'Sheet1!A:Z', // Mở rộng dải ô để tránh thiếu cột
+    });
+
+    const rows = res.data.values;
+
+    if (!rows || rows.length === 0) {
+      return response.status(200).send({ data: [], msg: "Sheet is empty" });
+    }
+
+    // 3. Xử lý dữ liệu
+    const headers = rows[0].map(h => h.toLowerCase().trim()); // Chuẩn hóa header (xóa khoảng trắng, viết thường)
+    const dataRows = rows.slice(1);
+
+    const ramProfiles = dataRows.map((row) => {
+      let obj = {};
+      headers.forEach((header, index) => {
+        let value = row[index];
+
+        // Ép kiểu số cho các trường định lượng
+        if (['size', 'speed', 'pc'].includes(header)) {
+          // Xử lý trường hợp giá trị có dấu phẩy hoặc khoảng trắng (vd: "3,200" hoặc "3 200")
+          const cleanNumber = value ? String(value).replace(/[, ]/g, '') : 0;
+          obj[header] = Number(cleanNumber) || 0;
+        } else {
+          obj[header] = value || ""; // Tránh trả về undefined
+        }
+      });
+      return obj;
+    });
+
+    return response.status(200).send({
+      data: ramProfiles,
+      msg: "Get data success"
+    });
+
+  } catch (err) {
+    console.error('Lỗi API Google Sheets:', err.message);
+
+    // Trả về lỗi chi tiết hơn để dễ debug
+    return response.status(500).send({
+      msg: "Internal Server Error",
+      error: err.message
+    });
+  }
+});