joseph
/
Log_service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change runtimeCheckLog

This commit is contained in:
joseph le 2023-10-04 11:15:31 +07:00
parent c6333d40ea
commit 6d04a73c35
1 changed files with 195 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

373
app/utils/runtimeCheckLogs.ts
View File

@ -87,13 +87,14 @@ export async function runtimeCheckLogs(folderPath) {
// Watch for changes in the files listed // Watch for changes in the files listed
async function watchFilesInList() { async function watchFilesInList() {
//only check new file ---> fileList - fileList_old = new file //only check new file ---> fileList - fileList_old = new file
let listFileWatch = fileList.slice(0, 40)?.filter((i) => fileList_old.includes(i) === false).map((file) => folderPath + "/" + file)
const watcher = chokidar.watch( const watcher = chokidar.watch(
fileList.slice(0,40) listFileWatch,
?.filter((i) => fileList_old.includes(i) === false) {
.map((file) => folderPath + "/" + file),{ persistent: true,
persistent: true, interval: 300000,
interval: 300000 }
}); );
watcher.setMaxListeners(200); watcher.setMaxListeners(200);
@ -101,97 +102,65 @@ export async function runtimeCheckLogs(folderPath) {
// fs.watchFile(filePath,{ interval: 15000 }, // fs.watchFile(filePath,{ interval: 15000 },
// async (eventType) => { // async (eventType) => {
//check special item, extra RAM, error in log //check special item, extra RAM, error in log
const fileName = path.split("/")[path.split("/").length - 1] const fileName = path.split("/")[path.split("/").length - 1];
const filePath = path const filePath = path;
let lines = []; let lines = [];
const today = DateTime.now().toFormat('yyyy-MM-dd'); const today = DateTime.now().toFormat("yyyy-MM-dd");
let allFile = await LogDetectFile.query() let allFile = await LogDetectFile.query().whereRaw(
.whereRaw(`DATE(created_at) = ?`, [today]) `DATE(created_at) = ?`,
// let allReport = await LogReport.all(); [today]
let allValue = await KeyValue.all(); );
const allReport = await LogReport.query() // let allReport = await LogReport.all();
.whereRaw(`DATE(created_at) = ?`, [today]) let allValue = await KeyValue.all();
const allReport = await LogReport.query().whereRaw(
`DATE(created_at) = ?`,
[today]
);
//get information file //get information file
let fileDetect = allFile?.filter( let fileDetect = allFile?.filter(
(i) => i.$original.file_name === fileName (i) => i.$original.file_name === fileName
)[0]; )[0];
let logsDetect = allReport?.filter(
(i) => i.$original.id_file === fileDetect?.id_ldf
);
//get the last line detected
let lastLine = Math.max(...logsDetect.map((obj) => obj.line));
//get content file in local let logsDetect = allReport?.filter(
let contentFile = await fs (i) => i.$original.id_file === fileDetect?.id_ldf
.readFileSync(filePath) );
.toString() //get the last line detected
?.split("\n"); let lastLine = Math.max(...logsDetect.map((obj) => obj.line));
//get index SN and send to ERP //get content file in local
let contentFile = await fs
.readFileSync(filePath)
.toString()
?.split("\n");
checkIndexSN(contentFile, lastLine, fileName); //get index SN and send to ERP
//get list item to check checkIndexSN(contentFile, lastLine, fileName);
let listKeyValues = allValue.filter(
(i) =>
i.$original.key === "MODEL_SPECIAL" ||
i.$original.key === "CATCH_FAULTY"
);
//get list exclude error //get list item to check
let listExcludeErr = allValue let listKeyValues = allValue.filter(
.filter((i) => i.$original.key === "EXCLUDE_ERR") (i) =>
.map((obj) => obj.$original.value); i.$original.key === "MODEL_SPECIAL" ||
i.$original.key === "CATCH_FAULTY"
);
//get list item special //get list exclude error
let listExtraItem = allValue let listExcludeErr = allValue
.filter((i) => i.$original.key === "MODEL_SPECIAL") .filter((i) => i.$original.key === "EXCLUDE_ERR")
.map((obj) => obj.$original.value); .map((obj) => obj.$original.value);
//Process file content //get list item special
if(contentFile.length>50000){ let listExtraItem = allValue
for (let i = 0; i < contentFile.length; i += 1000) { .filter((i) => i.$original.key === "MODEL_SPECIAL")
const chunk = contentFile.slice(i, i + 1000); .map((obj) => obj.$original.value);
chunk.map((line, index) => { //Process file content
//check line the line with errors and exclude errors if (contentFile.length > 50000) {
listKeyValues for (let i = 0; i < contentFile.length; i += 1000) {
.map((obj) => obj.$original.value) const chunk = contentFile.slice(i, i + 1000);
.map(async (value) => {
if ( chunk.map((line, index) => {
line.search(value) !== -1 &&
listExcludeErr.filter((err) => line.includes(err))
.length === 0
) {
let log = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
let checkLog = allReport?.filter(
(report) =>
report.$original.id_file === log?.id_ldf &&
report.$original.line === index + 1 &&
report.$original.detected_content === value
);
if (checkLog?.length === 0) {
await LogReport.create({
detected_content: value,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
}
// if(checkSpecialVersion())
});
});
}
}else{
contentFile.map((line, index) => {
//check line the line with errors and exclude errors //check line the line with errors and exclude errors
listKeyValues listKeyValues
.map((obj) => obj.$original.value) .map((obj) => obj.$original.value)
@ -204,14 +173,14 @@ export async function runtimeCheckLogs(folderPath) {
let log = allFile?.filter( let log = allFile?.filter(
(i) => i.$original.file_name === fileName (i) => i.$original.file_name === fileName
)[0]; )[0];
let checkLog = allReport?.filter( let checkLog = allReport?.filter(
(report) => (report) =>
report.$original.id_file === log?.id_ldf && report.$original.id_file === log?.id_ldf &&
report.$original.line === index + 1 && report.$original.line === index + 1 &&
report.$original.detected_content === value report.$original.detected_content === value
); );
if (checkLog?.length === 0) { if (checkLog?.length === 0) {
await LogReport.create({ await LogReport.create({
detected_content: value, detected_content: value,
@ -221,109 +190,157 @@ export async function runtimeCheckLogs(folderPath) {
lines.push(index + 1); lines.push(index + 1);
} }
} }
// if(checkSpecialVersion()) // if(checkSpecialVersion())
}); });
}); });
} }
} else {
contentFile.map((line, index) => {
//check line the line with errors and exclude errors
listKeyValues
.map((obj) => obj.$original.value)
.map(async (value) => {
if (
line.search(value) !== -1 &&
listExcludeErr.filter((err) => line.includes(err)).length ===
0
) {
let log = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
//true: import log to log_report table, send report to Zulip let checkLog = allReport?.filter(
setTimeout(async () => { (report) =>
if (lines.length === 0) { report.$original.id_file === log?.id_ldf &&
console.log(fileName + "has changed("+contentFile.length+") ---Good"); report.$original.line === index + 1 &&
} else { report.$original.detected_content === value
console.log( );
fileName + "has changed("+contentFile.length+") ---SOS---"+lines.length
);
let allReport_new = await LogReport.query()
.whereRaw(`DATE(created_at) = ?`, [today])
let fileDetect = allFile?.filter( if (checkLog?.length === 0) {
(i) => i.$original.file_name === fileName await LogReport.create({
)[0]; detected_content: value,
line: index + 1,
id_file: log?.id_ldf,
});
lines.push(index + 1);
}
}
let logsDetect = allReport_new?.filter( // if(checkSpecialVersion())
(i) => i.$original.id_file === fileDetect?.id_ldf
);
// console.log(logsDetect)
// await Database.rawQuery(
// "select * from log_reports where id_file = " +
// fileDetect?.id_ldf
// );
//Get all report newest
let listReport = await getListLineByItem(
logsDetect
.map((obj) => obj.$original)
.filter((l) => l.line > lastLine)
);
let content =
"| |Last updated at | Item/error | Line | Report \n|---|:---:|:---|:---|:-----------:\n";
let spoiler = "";
let issueFound = "";
listReport.map((log, index) => {
let item = listExtraItem.includes(log.detected_content)
? ":medal: **" + log.detected_content + "**"
: ":small_orange_diamond: " + log.detected_content;
log.line?.map((line) => {
issueFound =
issueFound +
"\n`" +
line +
"` " +
contentFile[line - 1]?.replace(
log.detected_content,
"[" +
log.detected_content +
"](https://logs.danielvu.com/logs/" +
fileName +
"#" +
line +
")"
);
});
content =
content +
"|" +
(index + 1) +
"|**" +
moment(Date.now()).format("HH:mm - DD/MM") +
"**|" +
item +
"|" +
log.line +
"|[View](https://logs.danielvu.com/logs/" +
fileName +
"#" +
log.line +
")\n";
}); });
});
}
sendMessToZulip( //true: import log to log_report table, send report to Zulip
"stream", setTimeout(async () => {
Env.get("ZULIP_STREAM_ALERT"), if (lines.length === 0) {
Env.get("ZULIP_TOPIC_ALERT"), console.log(
"------------\n\n:warning: :warning: **" + fileName + "has changed(" + contentFile.length + ") ---Good"
fileName + );
"**\n\n" + } else {
content + console.log(
"\n\n" + fileName +
spoiler + "has changed(" +
"\n\n***Issue found:***\n" + contentFile.length +
issueFound ") ---SOS---" +
); lines.length
} );
}, 3000) let allReport_new = await LogReport.query().whereRaw(
`DATE(created_at) = ?`,
[today]
);
let fileDetect = allFile?.filter(
(i) => i.$original.file_name === fileName
)[0];
let logsDetect = allReport_new?.filter(
(i) => i.$original.id_file === fileDetect?.id_ldf
);
// console.log(logsDetect)
// await Database.rawQuery(
// "select * from log_reports where id_file = " +
// fileDetect?.id_ldf
// );
//Get all report newest
let listReport = await getListLineByItem(
logsDetect
.map((obj) => obj.$original)
.filter((l) => l.line > lastLine)
);
let content =
"| |Last updated at | Item/error | Line | Report \n|---|:---:|:---|:---|:-----------:\n";
let spoiler = "";
let issueFound = "";
listReport.map((log, index) => {
let item = listExtraItem.includes(log.detected_content)
? ":medal: **" + log.detected_content + "**"
: ":small_orange_diamond: " + log.detected_content;
log.line?.map((line) => {
issueFound =
issueFound +
"\n`" +
line +
"` " +
contentFile[line - 1]?.replace(
log.detected_content,
"[" +
log.detected_content +
"](https://logs.danielvu.com/logs/" +
fileName +
"#" +
line +
")"
);
});
content =
content +
"|" +
(index + 1) +
"|**" +
moment(Date.now()).format("HH:mm - DD/MM") +
"**|" +
item +
"|" +
log.line +
"|[View](https://logs.danielvu.com/logs/" +
fileName +
"#" +
log.line +
")\n";
});
sendMessToZulip(
"stream",
Env.get("ZULIP_STREAM_ALERT"),
Env.get("ZULIP_TOPIC_ALERT"),
"------------\n\n:warning: :warning: **" +
fileName +
"**\n\n" +
content +
"\n\n" +
spoiler +
"\n\n***Issue found:***\n" +
issueFound
);
}
}, 3000);
// console.log(path + " change") // console.log(path + " change")
}); });
watcher.on("error", (error) => {
console.error(`Watcher error: ${error}`);
});
// await fileList.slice(0,40) // await fileList.slice(0,40)
// ?.filter((i) => fileList_old.includes(i) === false) // ?.filter((i) => fileList_old.includes(i) === false)
// ?.forEach((fileName) => { // ?.forEach((fileName) => {
// //path file // //path file
// const filePath = `${folderPath}/${fileName}`; // const filePath = `${folderPath}/${fileName}`;
// }); // });
} }
} catch (error) { } catch (error) {