336 lines
9.6 KiB
JavaScript
336 lines
9.6 KiB
JavaScript
// === app.js ===
|
|
const express = require("express");
|
|
const mysql = require("mysql2/promise");
|
|
const bodyParser = require("body-parser");
|
|
const axios = require("axios");
|
|
const cheerio = require("cheerio");
|
|
const dotenv = require("dotenv");
|
|
dotenv.config();
|
|
const app = express();
|
|
const PORT = 4000;
|
|
|
|
app.use(bodyParser.json());
|
|
app.use(express.static("public"));
|
|
|
|
// === MySQL Connection ===
|
|
const db = mysql.createPool({
|
|
host: "localhost",
|
|
user: "root",
|
|
password: "",
|
|
database: "log_analysis",
|
|
});
|
|
|
|
// === Create Table If Not Exists ===
|
|
(async () => {
|
|
await db.query(`
|
|
CREATE TABLE IF NOT EXISTS pid_logs (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
pid VARCHAR(100),
|
|
version VARCHAR(100),
|
|
log TEXT,
|
|
commands JSON,
|
|
errors LONGTEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
`);
|
|
})();
|
|
|
|
// === Regex-based command detection ===
|
|
function extractCommands(log) {
|
|
return {
|
|
inventory: /sh(ow)?\s+inv(entory)?/i.test(log),
|
|
version: /sh(ow)?\s+ver(sion)?/i.test(log),
|
|
license: /sh(ow)?\s+lic(ense)?/i.test(log),
|
|
logging: /sh(ow)?\s+log(ging)?/i.test(log),
|
|
};
|
|
}
|
|
|
|
function extractPIDVersion(log) {
|
|
const pidMatch = log.match(/PID:\s*([A-Z0-9\-]+)/);
|
|
const versionMatch = log.match(/Version\s+([\d\.A-Za-z\-]+)/);
|
|
return {
|
|
pid: pidMatch ? pidMatch[1] : "UNKNOWN",
|
|
version: versionMatch ? versionMatch[1] : "UNKNOWN",
|
|
};
|
|
}
|
|
|
|
function extractLogging(log) {
|
|
return log
|
|
.split("\n")
|
|
.filter(
|
|
(line) =>
|
|
line.startsWith("$ *") ||
|
|
line.startsWith("$ %") ||
|
|
line.startsWith("%") ||
|
|
line.includes("ALARM")
|
|
);
|
|
}
|
|
|
|
function extractRelevantLog(log, commands) {
|
|
const lines = log.split('\n');
|
|
const commandPatterns = [
|
|
{ key: 'inventory', regex: /sh(ow)?\s+inv(entory)?/i },
|
|
{ key: 'version', regex: /sh(ow)?\s+ver(sion)?/i },
|
|
{ key: 'license', regex: /sh(ow)?\s+lic(ense)?/i },
|
|
{ key: 'logging', regex: /sh(ow)?\s+log(ging)?/i },
|
|
];
|
|
|
|
let extracted = '';
|
|
const included = new Set();
|
|
|
|
for (let i = 0; i < lines.length; i++) {
|
|
for (const cmd of commandPatterns) {
|
|
const { key, regex } = cmd;
|
|
|
|
if (commands[key] && !included.has(key) && regex.test(lines[i])) {
|
|
included.add(key);
|
|
// console.log(log);
|
|
// Tách prompt và command
|
|
const match = lines[i].match(/^(.+?[#>])\s*(sh(ow)?\s+\w+)/i);
|
|
let prompt = '', command = '';
|
|
if (match) {
|
|
prompt = match[1]; // phần như "Switch#", "Router>", ...
|
|
command = match[2]; // phần như "show inventory", ...
|
|
}
|
|
console.log({prompt, command, lines: [lines[i], lines[i+1]]});
|
|
let block = [lines[i]];
|
|
|
|
for (let j = i + 1; j < lines.length; j++) {
|
|
// Nếu phát hiện dòng chứa command khác chưa được xử lý, thì dừng lại
|
|
// const isNextCommand = commandPatterns.some(c =>
|
|
// commands[c.key] && !included.has(c.key) && c.regex.test(lines[j])
|
|
// );
|
|
if (lines[j].includes(prompt)) break;
|
|
// if (isNextCommand) break;
|
|
|
|
block.push(lines[j]);
|
|
}
|
|
|
|
extracted += block.join('\n') + '\n';
|
|
}
|
|
}
|
|
}
|
|
|
|
return extracted.trim();
|
|
}
|
|
|
|
|
|
// === Search PID ===
|
|
app.get("/api/search", async (req, res) => {
|
|
const { pid } = req.query;
|
|
const [rows] = await db.query("SELECT * FROM pid_logs WHERE pid LIKE ?", [
|
|
`%${pid}%`,
|
|
]);
|
|
res.json(rows);
|
|
});
|
|
|
|
// === Add New Block ===
|
|
app.post("/api/add", async (req, res) => {
|
|
const { pid, version, log } = req.body;
|
|
const commands = extractCommands(log);
|
|
const errors = extractLogging(log);
|
|
const relevantLog = extractRelevantLog(log, commands);
|
|
await db.query(
|
|
`INSERT INTO pid_logs (pid, version, log, commands, errors) VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
pid,
|
|
version,
|
|
`[${filename}]\n${relevantLog}`,
|
|
JSON.stringify(commands),
|
|
errors.join('\n'),
|
|
]
|
|
);
|
|
|
|
res.json({ success: true });
|
|
});
|
|
|
|
// === Ask AI to extract errors ===
|
|
app.post("/api/extract-errors", async (req, res) => {
|
|
const { log } = req.body;
|
|
const response = await axios.post(
|
|
"https://api.openai.com/v1/chat/completions",
|
|
{
|
|
model: "gpt-4o-mini",
|
|
messages: [
|
|
{
|
|
role: "system",
|
|
content:
|
|
"Extract and summarize hardware or configuration errors from this Cisco log block.",
|
|
},
|
|
{ role: "user", content: log },
|
|
],
|
|
},
|
|
{
|
|
headers: {
|
|
Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
}
|
|
);
|
|
res.json({ errors: response.data.choices[0].message.content });
|
|
});
|
|
|
|
// Get versions for a specific PID
|
|
app.get("/api/pid/:pid/versions", async (req, res) => {
|
|
const { pid } = req.params;
|
|
const [rows] = await db.query(
|
|
"SELECT version, commands FROM pid_logs WHERE pid = ?",
|
|
[pid]
|
|
);
|
|
|
|
// Nhóm version nếu bị trùng, lấy unique
|
|
const seen = new Set();
|
|
const result = [];
|
|
|
|
for (const row of rows) {
|
|
if (seen.has(row.version)) continue;
|
|
seen.add(row.version);
|
|
result.push({
|
|
version: row.version,
|
|
commands: row.commands,
|
|
});
|
|
}
|
|
|
|
res.json(result);
|
|
});
|
|
|
|
|
|
// Get full log + errors by pid and version
|
|
app.get("/api/pid/:pid/version/:version", async (req, res) => {
|
|
const { pid, version } = req.params;
|
|
const [rows] = await db.query(
|
|
"SELECT log, errors FROM pid_logs WHERE pid = ? AND version = ?",
|
|
[pid, version]
|
|
);
|
|
if (rows.length === 0) return res.status(404).json({ message: "Not found" });
|
|
res.json(rows[0]);
|
|
});
|
|
|
|
app.get("/api/pids", async (req, res) => {
|
|
const [rows] = await db.query("SELECT DISTINCT pid FROM pid_logs ORDER BY pid");
|
|
res.json(rows.map((row) => row.pid));
|
|
});
|
|
|
|
// === Auto crawl logs from internal directory ===
|
|
async function fetchLogFile(url) {
|
|
const res = await axios.get(url);
|
|
return res.data;
|
|
}
|
|
|
|
async function crawlLogs() {
|
|
const BASE_URL = "http://172.16.5.7:8080";
|
|
console.log(`[${new Date().toISOString()}] Start crawl ${BASE_URL}`);
|
|
const res = await axios.get(BASE_URL);
|
|
const $ = cheerio.load(res.data);
|
|
const links = $("a")
|
|
.map((_, el) => $(el).attr("href"))
|
|
.get()
|
|
.filter((href) => href.endsWith(".log"));
|
|
|
|
const splitBlocks = (logContent) =>
|
|
logContent
|
|
.split(/(?=^.*?[#>]\s*sh(ow)?\s+inv(entory)?)/gim)
|
|
.map((b) => b?.trim())
|
|
.filter(Boolean);
|
|
|
|
for (const link of links) {
|
|
const fileUrl = `${BASE_URL}/${link}`;
|
|
const filename = link.replace(/\//g, "_");
|
|
// const [exists] = await db.query(
|
|
// "SELECT COUNT(*) as count FROM pid_logs WHERE log LIKE ?",
|
|
// [`%${filename}%`]
|
|
// );
|
|
// if (exists[0].count > 0) {
|
|
// console.log(`🟡 Skipped: ${filename}`);
|
|
// continue;
|
|
// }
|
|
|
|
try {
|
|
const log = await fetchLogFile(fileUrl);
|
|
const blocks = splitBlocks(log);
|
|
|
|
for (const block of blocks) {
|
|
const { pid, version } = extractPIDVersion(block);
|
|
const commands = extractCommands(block);
|
|
const errors = extractLogging(block);
|
|
|
|
if (pid === "UNKNOWN" || version === "UNKNOWN") continue;
|
|
|
|
const [existing] = await db.query(
|
|
"SELECT * FROM pid_logs WHERE pid = ? AND version = ?",
|
|
[pid, version]
|
|
);
|
|
|
|
// Nếu CHƯA CÓ -> INSERT mới toàn bộ
|
|
if (!existing[0]) {
|
|
const relevantLog = extractRelevantLog(block, commands);
|
|
|
|
await db.query(
|
|
`INSERT INTO pid_logs (pid, version, log, commands, errors) VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
pid,
|
|
version,
|
|
`[${filename}]\n${relevantLog}`,
|
|
JSON.stringify(commands),
|
|
errors.join('\n'),
|
|
]
|
|
);
|
|
|
|
console.log(`✅ Added ${pid} ${version} (${filename})`);
|
|
continue;
|
|
}
|
|
|
|
// Nếu ĐÃ CÓ -> Kiểm tra và UPDATE nếu cần
|
|
const oldCommands = existing[0].commands;
|
|
|
|
const isSameCommands = Object.keys(commands).every(
|
|
(key) => commands[key] === oldCommands[key]
|
|
);
|
|
|
|
if (isSameCommands) {
|
|
console.log(`🟡 Skipped existing PID+Version: ${pid} ${version}`);
|
|
continue;
|
|
}
|
|
|
|
const newCommands = {};
|
|
for (const key of Object.keys(commands)) {
|
|
if (commands[key] === true && oldCommands[key] === false) {
|
|
newCommands[key] = true;
|
|
}
|
|
}
|
|
|
|
if (Object.keys(newCommands).length === 0) {
|
|
console.log(`🟠 No new true commands → skip ${pid} ${version}`);
|
|
continue;
|
|
}
|
|
|
|
const newRelevantLog = extractRelevantLog(block, newCommands);
|
|
const updatedCommands = { ...oldCommands, ...newCommands };
|
|
const updatedLog = `${existing[0].log}\n[${filename}]\n${newRelevantLog}`;
|
|
|
|
await db.query(
|
|
`UPDATE pid_logs SET log = ?, commands = ?, errors = ? WHERE pid = ? AND version = ?`,
|
|
[
|
|
updatedLog,
|
|
JSON.stringify(updatedCommands),
|
|
`${existing[0].errors || ''}\n${errors.join('\n')}`.trim(),
|
|
pid,
|
|
version,
|
|
]
|
|
);
|
|
|
|
console.log(`🟢 Updated with new command(s) for ${pid} ${version}`);
|
|
}
|
|
} catch (err) {
|
|
console.error(`❌ Error ${filename}: ${err.message} ${err.stack}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// === Run crawl on start ===
|
|
// crawlLogs();
|
|
|
|
app.listen(PORT, () =>
|
|
console.log(`Server running at http://localhost:${PORT}`)
|
|
);
|