first commit
This commit is contained in:
Joseph
commit
1771027780
|
|
@ -0,0 +1 @@
|
|||
node_modules
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,134 @@
|
|||
const axios = require("axios");
|
||||
const cheerio = require("cheerio");
|
||||
const mysql = require("mysql2/promise");
|
||||
require("dotenv").config();
|
||||
|
||||
const db = mysql.createPool({
|
||||
host: "localhost",
|
||||
user: "root",
|
||||
password: "",
|
||||
database: "log_analysis",
|
||||
});
|
||||
|
||||
// === Helper functions ===
|
||||
function extractCommands(log) {
|
||||
return {
|
||||
inventory: /sh(ow)?\s+inv(entory)?/i.test(log),
|
||||
version: /sh(ow)?\s+ver(sion)?/i.test(log),
|
||||
license: /sh(ow)?\s+lic(ense)?/i.test(log),
|
||||
logging: /sh(ow)?\s+log(ging)?/i.test(log),
|
||||
};
|
||||
}
|
||||
|
||||
function extractPIDVersion(log) {
|
||||
const pidMatch = log.match(/PID:\s*([A-Z0-9\-]+)/);
|
||||
const versionMatch = log.match(/Version\s+([\d\.A-Za-z\-]+)/);
|
||||
return {
|
||||
pid: pidMatch ? pidMatch[1] : "UNKNOWN",
|
||||
version: versionMatch ? versionMatch[1] : "UNKNOWN",
|
||||
};
|
||||
}
|
||||
|
||||
function extractRelevantBlocksByCommand(log, commands) {
|
||||
const lines = log.split("\n");
|
||||
const commandPatterns = [
|
||||
{ key: "inventory", regex: /sh(ow)?\s+inv(entory)?/i },
|
||||
{ key: "version", regex: /sh(ow)?\s+ver(sion)?/i },
|
||||
{ key: "license", regex: /sh(ow)?\s+lic(ense)?/i },
|
||||
{ key: "logging", regex: /sh(ow)?\s+log(ging)?/i },
|
||||
];
|
||||
|
||||
const result = {};
|
||||
const included = new Set();
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
for (const { key, regex } of commandPatterns) {
|
||||
if (commands[key] && !included.has(key) && regex.test(lines[i])) {
|
||||
included.add(key);
|
||||
const match = lines[i].match(/^(.+?[#>])\s*(sh(ow)?\s+\w+)/i);
|
||||
const prompt = match?.[1] || "";
|
||||
const block = [lines[i]];
|
||||
for (let j = i + 1; j < lines.length; j++) {
|
||||
if (lines[j].includes(prompt)) break;
|
||||
block.push(lines[j]);
|
||||
}
|
||||
result[key] = block.join("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function fetchLogFile(url) {
|
||||
const res = await axios.get(url);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
async function crawlLogs(year = null) {
|
||||
const BASE_URL = "http://172.16.5.7:8080";
|
||||
console.log(`[${new Date().toISOString()}] Start crawl ${BASE_URL}`);
|
||||
|
||||
const res = await axios.get(BASE_URL);
|
||||
const $ = cheerio.load(res.data);
|
||||
const links = $("a")
|
||||
.map((_, el) => $(el).attr("href"))
|
||||
.get()
|
||||
.filter((href) => href.endsWith(".log"));
|
||||
|
||||
const filteredLinks = year
|
||||
? links.filter((l) => l.includes(`${year}`))
|
||||
: links;
|
||||
|
||||
const splitBlocks = (logContent) =>
|
||||
logContent
|
||||
.split(/(?=^.*?[#>]\s*sh(ow)?\s+inv(entory)?)/gim)
|
||||
.map((b) => b?.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const link of filteredLinks) {
|
||||
const fileUrl = `${BASE_URL}/${link}`;
|
||||
const filename = link.replace(/\//g, "_");
|
||||
|
||||
try {
|
||||
const log = await fetchLogFile(fileUrl);
|
||||
const blocks = splitBlocks(log);
|
||||
|
||||
for (const block of blocks) {
|
||||
const { pid, version } = extractPIDVersion(block);
|
||||
const commands = extractCommands(block);
|
||||
if (pid === "UNKNOWN" || version === "UNKNOWN") continue;
|
||||
|
||||
const [[device]] = await db.query(
|
||||
`SELECT id FROM devices WHERE pid = ? AND version = ?`,
|
||||
[pid, version]
|
||||
);
|
||||
let deviceId = device?.id;
|
||||
if (!deviceId) {
|
||||
const result = await db.query(
|
||||
`INSERT INTO devices (pid, version) VALUES (?, ?)`,
|
||||
[pid, version]
|
||||
);
|
||||
deviceId = result[0].insertId;
|
||||
}
|
||||
|
||||
const commandBlocks = extractRelevantBlocksByCommand(block, commands);
|
||||
for (const [command, output] of Object.entries(commandBlocks)) {
|
||||
const table = `${command}_outputs`;
|
||||
await db.query(
|
||||
`INSERT INTO ${table} (device_id, filename, output) VALUES (?, ?, ?)`,
|
||||
[deviceId, filename, output]
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`✅ ${pid} ${version} - ${filename}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`❌ ${filename}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 👇 Call from command line
|
||||
const yearArg = process.argv[2]; // e.g. node crawl.js 2023
|
||||
crawlLogs(yearArg);
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,36 @@
|
|||
{
|
||||
"name": "network-log-analyzer",
|
||||
"version": "1.0.0",
|
||||
"description": "Network device log analyzer and aggregator",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "nodemon server.js",
|
||||
"dev": "nodemon server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.10.0",
|
||||
"better-sqlite3": "^12.2.0",
|
||||
"cheerio": "^1.1.1",
|
||||
"cli-progress": "^3.12.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.0",
|
||||
"express": "^4.18.2",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
"mysql2": "^3.14.2",
|
||||
"sqlite3": "^5.1.6",
|
||||
"string-similarity": "^4.0.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.1.10"
|
||||
},
|
||||
"keywords": [
|
||||
"network",
|
||||
"log",
|
||||
"analyzer",
|
||||
"cisco",
|
||||
"juniper",
|
||||
"networking"
|
||||
],
|
||||
"author": "Network Team",
|
||||
"license": "MIT"
|
||||
}
|
||||
|
|
@ -0,0 +1,522 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>PID Log Viewer</title>
|
||||
<style>
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
:root {
|
||||
--primary: #1e88e5;
|
||||
--primary-light: #e3f2fd;
|
||||
--gray-light: #f8f9fa;
|
||||
--gray-border: #e0e0e0;
|
||||
--text-color: #333;
|
||||
--bg-color: #ffffff;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: "Segoe UI", Roboto, sans-serif;
|
||||
display: flex;
|
||||
height: 100vh;
|
||||
background-color: var(--bg-color);
|
||||
color: var(--text-color);
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
width: 280px;
|
||||
background-color: var(--gray-light);
|
||||
border-right: 1px solid var(--gray-border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.sidebar-header {
|
||||
padding: 16px;
|
||||
background: white;
|
||||
border-bottom: 1px solid #ddd;
|
||||
flex-shrink: 0;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.sidebar-list {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 12px 16px;
|
||||
}
|
||||
|
||||
.pid-button {
|
||||
display: block;
|
||||
width: 100%;
|
||||
padding: 8px 12px;
|
||||
margin: 4px 0;
|
||||
text-align: left;
|
||||
border: 1px solid transparent;
|
||||
background: #fff;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.pid-button:hover {
|
||||
background-color: var(--primary-light);
|
||||
border-color: var(--primary);
|
||||
color: var(--primary);
|
||||
}
|
||||
|
||||
.pid-button.selected {
|
||||
background-color: var(--primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.content {
|
||||
flex: 1;
|
||||
padding: 24px;
|
||||
overflow-y: auto;
|
||||
background-color: #fefefe;
|
||||
}
|
||||
|
||||
select,
|
||||
input[type="text"] {
|
||||
width: 100%;
|
||||
padding: 6px;
|
||||
/* margin-bottom: 16px; */
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 6px;
|
||||
font-size: 14px;
|
||||
outline: none;
|
||||
transition: border-color 0.2s;
|
||||
}
|
||||
|
||||
select:focus,
|
||||
input[type="text"]:focus {
|
||||
border-color: var(--primary);
|
||||
}
|
||||
|
||||
.log-block {
|
||||
background: #fafafa;
|
||||
padding: 12px;
|
||||
margin-bottom: 12px;
|
||||
border-left: 4px solid var(--primary);
|
||||
font-family: monospace;
|
||||
white-space: pre-wrap;
|
||||
border-radius: 6px;
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.04);
|
||||
}
|
||||
|
||||
.command-section {
|
||||
margin-top: 32px;
|
||||
}
|
||||
|
||||
.command-section h3 {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 12px;
|
||||
font-size: 17px;
|
||||
}
|
||||
|
||||
.command-section h3 > div {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.next-btn {
|
||||
font-size: 13px;
|
||||
background: var(--primary);
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 5px 10px;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s ease;
|
||||
}
|
||||
|
||||
.next-btn:hover:not(:disabled) {
|
||||
background-color: #1565c0;
|
||||
}
|
||||
|
||||
.next-btn:disabled {
|
||||
background-color: #ccc;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08);
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
table thead {
|
||||
background: var(--primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
table th,
|
||||
table td {
|
||||
padding: 12px 14px;
|
||||
text-align: left;
|
||||
font-size: 14px;
|
||||
border-bottom: 1px solid #eee;
|
||||
}
|
||||
|
||||
table tbody tr:hover {
|
||||
background-color: #f1f8ff;
|
||||
}
|
||||
|
||||
code {
|
||||
background: #f4f4f4;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
mark {
|
||||
background-color: #ffdb38;
|
||||
padding: 0 2px;
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
#selectAndVersionDiv {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
#selectedPIDDiv {
|
||||
width: 23%;
|
||||
}
|
||||
#selectedVersionDiv {
|
||||
width: 53%;
|
||||
}
|
||||
|
||||
.action-btn {
|
||||
background-color: #8d7df3;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 8px 14px;
|
||||
margin: 5px;
|
||||
border-radius: 6px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s ease, box-shadow 0.2s ease;
|
||||
box-shadow: 0 2px 4px rgba(0, 123, 255, 0.2);
|
||||
}
|
||||
|
||||
.action-btn:hover {
|
||||
background-color: #0056b3;
|
||||
box-shadow: 0 4px 8px rgba(0, 123, 255, 0.3);
|
||||
}
|
||||
|
||||
.action-btn:active {
|
||||
background-color: #004a99;
|
||||
transform: scale(0.98);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="sidebar">
|
||||
<div class="sidebar-header">
|
||||
<h3>PIDs (<span id="pidCount">0</span>)</h3>
|
||||
<input type="text" id="pidSearch" placeholder="🔍 Search PID..." />
|
||||
</div>
|
||||
<div class="sidebar-list" id="pidListQuick"></div>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<!-- <h3>📊 PID Log Viewer</h3> -->
|
||||
<div id="selectAndVersionDiv">
|
||||
<div id="selectedPIDDiv">
|
||||
<label for="pidList">PID</label>
|
||||
<select id="pidList" disabled></select>
|
||||
</div>
|
||||
<div id="selectedVersionDiv">
|
||||
<label for="versionList">Select Version</label>
|
||||
<select id="versionList"></select>
|
||||
</div>
|
||||
|
||||
<div style="width: 23%; align-items: center">
|
||||
<button id="copyBtn" class="action-btn">📋 Copy</button>
|
||||
<button id="downloadBtn" class="action-btn">⬇️ Download</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="errorTableContainer" class="command-section">
|
||||
<h3>⚠️ Error History for PID</h3>
|
||||
<table border="1" cellspacing="0" cellpadding="5" width="100%">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="text-align: left">Logging</th>
|
||||
<th style="text-align: left">Pattern</th>
|
||||
<th style="text-align: left">Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="errorTableBody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div id="logSections"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const pidSearchInput = document.getElementById("pidSearch");
|
||||
const pidList = document.getElementById("pidList");
|
||||
const versionList = document.getElementById("versionList");
|
||||
const pidListQuick = document.getElementById("pidListQuick");
|
||||
const logSections = document.getElementById("logSections");
|
||||
const commands = ["inventory", "version", "license", "logging"];
|
||||
|
||||
const logsCache = {}; // Store logs and current index for each command
|
||||
|
||||
async function loadPIDs() {
|
||||
const res = await fetch("/api/pids");
|
||||
const pids = await res.json();
|
||||
document.getElementById("pidCount").textContent = pids.length;
|
||||
pidListQuick.innerHTML = "";
|
||||
|
||||
pids.forEach((pid) => {
|
||||
const btn = document.createElement("button");
|
||||
btn.className = "pid-button";
|
||||
btn.textContent = pid;
|
||||
btn.onclick = () => selectPID(pid);
|
||||
pidListQuick.appendChild(btn);
|
||||
});
|
||||
}
|
||||
|
||||
async function selectPID(pid) {
|
||||
document
|
||||
.querySelectorAll(".pid-button")
|
||||
.forEach((b) => b.classList.remove("selected"));
|
||||
const target = [...document.querySelectorAll(".pid-button")].find(
|
||||
(b) => b.textContent === pid
|
||||
);
|
||||
if (target) target.classList.add("selected");
|
||||
|
||||
const res = await fetch(`/api/pid/${encodeURIComponent(pid)}/versions`);
|
||||
const versions = await res.json();
|
||||
|
||||
pidList.innerHTML = `<option value="${pid}">${pid}</option>`;
|
||||
versionList.innerHTML = versions
|
||||
.map((v) => {
|
||||
const cmds = Object.entries(v.commands)
|
||||
.filter(([_, count]) => count > 0)
|
||||
.map(([cmd, count]) => `${cmd} (${count})`)
|
||||
.join(", ");
|
||||
return `<option value="${v.version}">${v.version} - ${
|
||||
cmds || "No commands"
|
||||
}</option>`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
// ✅ Default chọn version đầu tiên
|
||||
if (versions.length > 0) {
|
||||
versionList.value = versions[0].version;
|
||||
versionList.dispatchEvent(new Event("change"));
|
||||
}
|
||||
|
||||
await loadErrorTable(pid);
|
||||
}
|
||||
|
||||
pidSearchInput.addEventListener("input", async () => {
|
||||
const keyword = pidSearchInput.value.toLowerCase().trim();
|
||||
const res = await fetch("/api/pids");
|
||||
const pids = await res.json();
|
||||
const filtered = pids.filter((pid) =>
|
||||
pid.toLowerCase().includes(keyword)
|
||||
);
|
||||
|
||||
pidListQuick.innerHTML = "";
|
||||
filtered.forEach((pid) => {
|
||||
const btn = document.createElement("button");
|
||||
btn.className = "pid-button";
|
||||
btn.textContent = pid;
|
||||
btn.onclick = () => selectPID(pid);
|
||||
pidListQuick.appendChild(btn);
|
||||
});
|
||||
|
||||
pidList.innerHTML = filtered
|
||||
.map((pid) => `<option value="${pid}">${pid}</option>`)
|
||||
.join("");
|
||||
if (filtered.length > 0) {
|
||||
pidList.value = filtered[0];
|
||||
selectPID(filtered[0]);
|
||||
}
|
||||
});
|
||||
|
||||
pidList.addEventListener("change", () => {
|
||||
const pid = pidList.value;
|
||||
if (pid) selectPID(pid);
|
||||
});
|
||||
|
||||
versionList.addEventListener("change", () => {
|
||||
const pid = pidList.value;
|
||||
const version = versionList.value;
|
||||
if (pid && version) loadLogs(pid, version);
|
||||
});
|
||||
|
||||
async function loadLogs(pid, version) {
|
||||
logSections.innerHTML = "";
|
||||
logsCache[`${pid}_${version}`] = {};
|
||||
|
||||
for (const cmd of commands) {
|
||||
const res = await fetch(
|
||||
`/api/device/${encodeURIComponent(pid)}/${encodeURIComponent(
|
||||
version
|
||||
)}/${cmd}`
|
||||
);
|
||||
const data = await res.json();
|
||||
if (!data || data.length === 0) continue;
|
||||
|
||||
logsCache[`${pid}_${version}`][cmd] = {
|
||||
logs: data,
|
||||
index: 0,
|
||||
};
|
||||
|
||||
const section = document.createElement("div");
|
||||
section.className = "command-section";
|
||||
section.id = `section-${cmd}`;
|
||||
section.innerHTML = `
|
||||
<h3>
|
||||
${"show " + cmd} (${data.length})
|
||||
<div>
|
||||
<button class="next-btn" id="btn-prev-${cmd}">Previous</button>
|
||||
<span id="page-info-${cmd}">Output 1/${data.length}</span>
|
||||
<button class="next-btn" id="btn-next-${cmd}">Next</button>
|
||||
</div>
|
||||
</h3>
|
||||
<div class="log-block" id="log-${cmd}">
|
||||
<b>${data[0].filename}</b>\n${data[0].output}
|
||||
</div>
|
||||
`;
|
||||
|
||||
logSections.appendChild(section);
|
||||
|
||||
// Navigation logic
|
||||
const updateDisplay = () => {
|
||||
const info = logsCache[`${pid}_${version}`][cmd];
|
||||
const curr = info.logs[info.index];
|
||||
document.getElementById(
|
||||
`log-${cmd}`
|
||||
).innerHTML = `<b>${curr.filename}</b>\n${curr.output}`;
|
||||
document.getElementById(`page-info-${cmd}`).textContent = `Output ${
|
||||
info.index + 1
|
||||
}/${info.logs.length}`;
|
||||
document.getElementById(`btn-prev-${cmd}`).disabled =
|
||||
info.index === 0;
|
||||
};
|
||||
|
||||
document
|
||||
.getElementById(`btn-next-${cmd}`)
|
||||
.addEventListener("click", () => {
|
||||
const info = logsCache[`${pid}_${version}`][cmd];
|
||||
info.index = (info.index + 1) % info.logs.length;
|
||||
updateDisplay();
|
||||
});
|
||||
|
||||
document
|
||||
.getElementById(`btn-prev-${cmd}`)
|
||||
.addEventListener("click", () => {
|
||||
const info = logsCache[`${pid}_${version}`][cmd];
|
||||
info.index =
|
||||
(info.index - 1 + info.logs.length) % info.logs.length;
|
||||
updateDisplay();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
loadPIDs();
|
||||
|
||||
const copyBtn = document.getElementById("copyBtn");
|
||||
const downloadBtn = document.getElementById("downloadBtn");
|
||||
|
||||
function getExportText() {
|
||||
const pid = pidList.value || "";
|
||||
const version = versionList.value || "";
|
||||
const inventory = getCommandOutput("inventory");
|
||||
const versionOut = getCommandOutput("version");
|
||||
const logging = getCommandOutput("logging");
|
||||
const license = getCommandOutput("license");
|
||||
|
||||
return `PID: ${pid}
|
||||
Version: ${version}
|
||||
|
||||
${inventory}
|
||||
|
||||
${versionOut}
|
||||
|
||||
${logging}
|
||||
|
||||
${license}`.trim();
|
||||
}
|
||||
|
||||
// Assume you store outputs in memory or fetch based on UI state
|
||||
function getCommandOutput(command) {
|
||||
const pid = pidList.value;
|
||||
const version = versionList.value;
|
||||
const key = `${pid}_${version}`;
|
||||
|
||||
if (
|
||||
logsCache[key] &&
|
||||
logsCache[key][command] &&
|
||||
logsCache[key][command].logs.length > 0
|
||||
) {
|
||||
const { logs, index } = logsCache[key][command];
|
||||
return logs[index].output || "";
|
||||
}
|
||||
|
||||
return "No data";
|
||||
}
|
||||
|
||||
copyBtn.addEventListener("click", async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(getExportText());
|
||||
alert("Copied to clipboard!");
|
||||
} catch (err) {
|
||||
alert("Failed to copy.");
|
||||
}
|
||||
});
|
||||
|
||||
downloadBtn.addEventListener("click", () => {
|
||||
const content = getExportText();
|
||||
const blob = new Blob([content], { type: "text/plain" });
|
||||
const a = document.createElement("a");
|
||||
a.href = URL.createObjectURL(blob);
|
||||
a.download = `log-${pidList.value || "unknown"}.txt`;
|
||||
a.click();
|
||||
});
|
||||
|
||||
async function loadErrorTable(pid) {
|
||||
const res = await fetch(`/api/errors/${encodeURIComponent(pid)}`);
|
||||
const data = await res.json();
|
||||
|
||||
const tbody = document.getElementById("errorTableBody");
|
||||
tbody.innerHTML = "";
|
||||
|
||||
if (data.length === 0) {
|
||||
tbody.innerHTML = `<tr><td colspan="5">✅ No known issues for this PID</td></tr>`;
|
||||
return;
|
||||
}
|
||||
|
||||
for (const err of data) {
|
||||
const tr = document.createElement("tr");
|
||||
tr.innerHTML = `
|
||||
<td><code>${err.highlighted_message}</code></td>
|
||||
<td><code>${err.regex_pattern}</code></td>
|
||||
<td><code>${err.description}</code></td>
|
||||
`;
|
||||
tbody.appendChild(tr);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
const express = require("express");
|
||||
const mysql = require("mysql2/promise");
|
||||
const bodyParser = require("body-parser");
|
||||
const dotenv = require("dotenv");
|
||||
dotenv.config();
|
||||
|
||||
const stringSimilarity = require('string-similarity');
|
||||
|
||||
// Nhận mảng message đầu vào, trả về mảng message đã loại bỏ trùng lặp
|
||||
function deduplicateErrors(errors, threshold = 0.3) {
|
||||
const unique = [];
|
||||
|
||||
for (const err of errors) {
|
||||
const isDuplicate = unique.some(u => {
|
||||
const similarity = stringSimilarity.compareTwoStrings(err.error_message, u.error_message);
|
||||
return similarity >= threshold;
|
||||
});
|
||||
|
||||
if (!isDuplicate) {
|
||||
unique.push(err);
|
||||
}
|
||||
}
|
||||
|
||||
return unique;
|
||||
}
|
||||
|
||||
|
||||
const app = express();
|
||||
const PORT = 4000;
|
||||
|
||||
app.use(bodyParser.json());
|
||||
app.use(express.static("public"));
|
||||
|
||||
const db = mysql.createPool({
|
||||
host: "localhost",
|
||||
user: "root",
|
||||
password: "",
|
||||
database: "log_analysis",
|
||||
});
|
||||
|
||||
// === Tạo bảng
|
||||
(async () => {
|
||||
await db.query(`
|
||||
CREATE TABLE IF NOT EXISTS devices (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
pid VARCHAR(100) NOT NULL,
|
||||
version VARCHAR(100) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uniq_pid_version (pid, version),
|
||||
INDEX idx_pid (pid),
|
||||
INDEX idx_version (version),
|
||||
INDEX idx_created (created_at)
|
||||
)
|
||||
`);
|
||||
|
||||
const createOutputTable = async (command) => {
|
||||
await db.query(`
|
||||
CREATE TABLE IF NOT EXISTS ${command}_outputs (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
device_id INT NOT NULL,
|
||||
filename VARCHAR(255),
|
||||
output TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
INDEX idx_device_created (device_id, created_at),
|
||||
FOREIGN KEY (device_id) REFERENCES devices(id) ON DELETE CASCADE
|
||||
)
|
||||
`);
|
||||
};
|
||||
|
||||
await createOutputTable("inventory");
|
||||
await createOutputTable("version");
|
||||
await createOutputTable("license");
|
||||
await createOutputTable("logging");
|
||||
})();
|
||||
|
||||
// === APIs ===
|
||||
|
||||
app.get("/api/pids", async (req, res) => {
|
||||
const [rows] = await db.query("SELECT DISTINCT pid FROM devices ORDER BY pid");
|
||||
res.json(rows.map((r) => r.pid));
|
||||
});
|
||||
|
||||
app.get("/api/pid/:pid/versions", async (req, res) => {
|
||||
const { pid } = req.params;
|
||||
|
||||
// Lấy danh sách các version tương ứng với PID
|
||||
const [devices] = await db.query(
|
||||
"SELECT id, version FROM devices WHERE pid = ?",
|
||||
[pid]
|
||||
);
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const device of devices) {
|
||||
const deviceId = device.id;
|
||||
const version = device.version;
|
||||
|
||||
const [[inv]] = await db.query("SELECT COUNT(*) AS c FROM inventory_outputs WHERE device_id = ?", [deviceId]);
|
||||
const [[ver]] = await db.query("SELECT COUNT(*) AS c FROM version_outputs WHERE device_id = ?", [deviceId]);
|
||||
const [[lic]] = await db.query("SELECT COUNT(*) AS c FROM license_outputs WHERE device_id = ?", [deviceId]);
|
||||
const [[log]] = await db.query("SELECT COUNT(*) AS c FROM logging_outputs WHERE device_id = ?", [deviceId]);
|
||||
|
||||
results.push({
|
||||
version,
|
||||
commands: {
|
||||
inventory: inv.c,
|
||||
version: ver.c,
|
||||
license: lic.c,
|
||||
logging: log.c,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
res.json(results);
|
||||
});
|
||||
|
||||
|
||||
|
||||
app.get("/api/device/:pid/:version/:command", async (req, res) => {
|
||||
const { pid, version, command } = req.params;
|
||||
|
||||
const [[device]] = await db.query(
|
||||
`SELECT id FROM devices WHERE pid = ? AND version = ?`,
|
||||
[pid, version]
|
||||
);
|
||||
if (!device) return res.status(404).json({ message: "Not found" });
|
||||
|
||||
const [logs] = await db.query(
|
||||
`SELECT filename, output, created_at FROM ${command}_outputs WHERE device_id = ? ORDER BY created_at DESC`,
|
||||
[device.id]
|
||||
);
|
||||
|
||||
res.json(logs);
|
||||
});
|
||||
|
||||
// Danh sách regex lọc lỗi
|
||||
const errorPatterns = [
|
||||
{
|
||||
regex: /%\w+-[123]-\w+:/i,
|
||||
description: 'Lỗi mức độ 1–3 (critical, error, alert) trong định dạng syslog Cisco',
|
||||
},
|
||||
{
|
||||
regex: /%\w+-4-\w+:/i,
|
||||
description: 'Lỗi mức độ cảnh báo (level 4 - warning)',
|
||||
},
|
||||
{
|
||||
regex: /(fail|error|unexpected|badcfg|invalid|denied|disabled|timeout|panic)/i,
|
||||
description: 'Từ khóa phổ biến của lỗi trong nội dung log',
|
||||
},
|
||||
{
|
||||
regex: /^\d{1,2}:\d{2}:\d{2}:.*?%\w+-[1-4]-\w+:/i,
|
||||
description: 'Dòng log bắt đầu bằng timestamp có mã lỗi Cisco',
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
// API: Trả về các dòng lỗi từ bảng logging_outputs theo PID
|
||||
app.get('/api/errors/:pid', async (req, res) => {
|
||||
const pid = req.params.pid;
|
||||
|
||||
try {
|
||||
const [rows] = await db.execute(`
|
||||
SELECT lo.output, d.pid, lo.filename
|
||||
FROM logging_outputs lo
|
||||
JOIN devices d ON lo.device_id = d.id
|
||||
WHERE d.pid = ?
|
||||
`, [pid]);
|
||||
|
||||
const matchedLines = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const lines = row.output.split(/\r?\n/);
|
||||
|
||||
for (const line of lines.filter((l) => l.includes("*") || l.includes("%"))) {
|
||||
for (const pattern of errorPatterns) {
|
||||
const regex = new RegExp(pattern.regex.source, 'gi');
|
||||
if (regex.test(line)) {
|
||||
// Reset regex to start (because `.test()` may advance position in some cases)
|
||||
regex.lastIndex = 0;
|
||||
const highlighted = line.replace(regex, match => `<mark>${match}</mark>`);
|
||||
|
||||
matchedLines.push({
|
||||
pid: row.pid,
|
||||
filename: row.filename,
|
||||
error_message: line.trim(),
|
||||
highlighted_message: highlighted.trim(),
|
||||
regex_pattern: pattern.regex.source,
|
||||
description: pattern.description,
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const filtered = deduplicateErrors(matchedLines, 0.6);
|
||||
res.json(filtered);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/test', async (req, res) => {
|
||||
console.log(req)
|
||||
res.send("ok")
|
||||
})
|
||||
|
||||
app.listen(PORT, () =>
|
||||
console.log(`🚀 Server running at http://localhost:${PORT}`)
|
||||
);
|
||||
|
|
@ -0,0 +1,335 @@
|
|||
// === app.js ===
|
||||
const express = require("express");
|
||||
const mysql = require("mysql2/promise");
|
||||
const bodyParser = require("body-parser");
|
||||
const axios = require("axios");
|
||||
const cheerio = require("cheerio");
|
||||
const dotenv = require("dotenv");
|
||||
dotenv.config();
|
||||
const app = express();
|
||||
const PORT = 4000;
|
||||
|
||||
app.use(bodyParser.json());
|
||||
app.use(express.static("public"));
|
||||
|
||||
// === MySQL Connection ===
|
||||
const db = mysql.createPool({
|
||||
host: "localhost",
|
||||
user: "root",
|
||||
password: "",
|
||||
database: "log_analysis",
|
||||
});
|
||||
|
||||
// === Create Table If Not Exists ===
|
||||
(async () => {
|
||||
await db.query(`
|
||||
CREATE TABLE IF NOT EXISTS pid_logs (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
pid VARCHAR(100),
|
||||
version VARCHAR(100),
|
||||
log TEXT,
|
||||
commands JSON,
|
||||
errors LONGTEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
`);
|
||||
})();
|
||||
|
||||
// === Regex-based command detection ===
|
||||
function extractCommands(log) {
|
||||
return {
|
||||
inventory: /sh(ow)?\s+inv(entory)?/i.test(log),
|
||||
version: /sh(ow)?\s+ver(sion)?/i.test(log),
|
||||
license: /sh(ow)?\s+lic(ense)?/i.test(log),
|
||||
logging: /sh(ow)?\s+log(ging)?/i.test(log),
|
||||
};
|
||||
}
|
||||
|
||||
function extractPIDVersion(log) {
|
||||
const pidMatch = log.match(/PID:\s*([A-Z0-9\-]+)/);
|
||||
const versionMatch = log.match(/Version\s+([\d\.A-Za-z\-]+)/);
|
||||
return {
|
||||
pid: pidMatch ? pidMatch[1] : "UNKNOWN",
|
||||
version: versionMatch ? versionMatch[1] : "UNKNOWN",
|
||||
};
|
||||
}
|
||||
|
||||
function extractLogging(log) {
|
||||
return log
|
||||
.split("\n")
|
||||
.filter(
|
||||
(line) =>
|
||||
line.startsWith("$ *") ||
|
||||
line.startsWith("$ %") ||
|
||||
line.startsWith("%") ||
|
||||
line.includes("ALARM")
|
||||
);
|
||||
}
|
||||
|
||||
function extractRelevantLog(log, commands) {
|
||||
const lines = log.split('\n');
|
||||
const commandPatterns = [
|
||||
{ key: 'inventory', regex: /sh(ow)?\s+inv(entory)?/i },
|
||||
{ key: 'version', regex: /sh(ow)?\s+ver(sion)?/i },
|
||||
{ key: 'license', regex: /sh(ow)?\s+lic(ense)?/i },
|
||||
{ key: 'logging', regex: /sh(ow)?\s+log(ging)?/i },
|
||||
];
|
||||
|
||||
let extracted = '';
|
||||
const included = new Set();
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
for (const cmd of commandPatterns) {
|
||||
const { key, regex } = cmd;
|
||||
|
||||
if (commands[key] && !included.has(key) && regex.test(lines[i])) {
|
||||
included.add(key);
|
||||
// console.log(log);
|
||||
// Tách prompt và command
|
||||
const match = lines[i].match(/^(.+?[#>])\s*(sh(ow)?\s+\w+)/i);
|
||||
let prompt = '', command = '';
|
||||
if (match) {
|
||||
prompt = match[1]; // phần như "Switch#", "Router>", ...
|
||||
command = match[2]; // phần như "show inventory", ...
|
||||
}
|
||||
console.log({prompt, command, lines: [lines[i], lines[i+1]]});
|
||||
let block = [lines[i]];
|
||||
|
||||
for (let j = i + 1; j < lines.length; j++) {
|
||||
// Nếu phát hiện dòng chứa command khác chưa được xử lý, thì dừng lại
|
||||
// const isNextCommand = commandPatterns.some(c =>
|
||||
// commands[c.key] && !included.has(c.key) && c.regex.test(lines[j])
|
||||
// );
|
||||
if (lines[j].includes(prompt)) break;
|
||||
// if (isNextCommand) break;
|
||||
|
||||
block.push(lines[j]);
|
||||
}
|
||||
|
||||
extracted += block.join('\n') + '\n';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return extracted.trim();
|
||||
}
|
||||
|
||||
|
||||
// === Search PID ===
|
||||
app.get("/api/search", async (req, res) => {
|
||||
const { pid } = req.query;
|
||||
const [rows] = await db.query("SELECT * FROM pid_logs WHERE pid LIKE ?", [
|
||||
`%${pid}%`,
|
||||
]);
|
||||
res.json(rows);
|
||||
});
|
||||
|
||||
// === Add New Block ===
|
||||
app.post("/api/add", async (req, res) => {
|
||||
const { pid, version, log } = req.body;
|
||||
const commands = extractCommands(log);
|
||||
const errors = extractLogging(log);
|
||||
const relevantLog = extractRelevantLog(log, commands);
|
||||
await db.query(
|
||||
`INSERT INTO pid_logs (pid, version, log, commands, errors) VALUES (?, ?, ?, ?, ?)`,
|
||||
[
|
||||
pid,
|
||||
version,
|
||||
`[${filename}]\n${relevantLog}`,
|
||||
JSON.stringify(commands),
|
||||
errors.join('\n'),
|
||||
]
|
||||
);
|
||||
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
||||
// === Ask AI to extract errors ===
|
||||
app.post("/api/extract-errors", async (req, res) => {
|
||||
const { log } = req.body;
|
||||
const response = await axios.post(
|
||||
"https://api.openai.com/v1/chat/completions",
|
||||
{
|
||||
model: "gpt-4o-mini",
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"Extract and summarize hardware or configuration errors from this Cisco log block.",
|
||||
},
|
||||
{ role: "user", content: log },
|
||||
],
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
);
|
||||
res.json({ errors: response.data.choices[0].message.content });
|
||||
});
|
||||
|
||||
// Get versions for a specific PID
|
||||
app.get("/api/pid/:pid/versions", async (req, res) => {
|
||||
const { pid } = req.params;
|
||||
const [rows] = await db.query(
|
||||
"SELECT version, commands FROM pid_logs WHERE pid = ?",
|
||||
[pid]
|
||||
);
|
||||
|
||||
// Nhóm version nếu bị trùng, lấy unique
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
|
||||
for (const row of rows) {
|
||||
if (seen.has(row.version)) continue;
|
||||
seen.add(row.version);
|
||||
result.push({
|
||||
version: row.version,
|
||||
commands: row.commands,
|
||||
});
|
||||
}
|
||||
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
|
||||
// Get full log + errors by pid and version
|
||||
app.get("/api/pid/:pid/version/:version", async (req, res) => {
|
||||
const { pid, version } = req.params;
|
||||
const [rows] = await db.query(
|
||||
"SELECT log, errors FROM pid_logs WHERE pid = ? AND version = ?",
|
||||
[pid, version]
|
||||
);
|
||||
if (rows.length === 0) return res.status(404).json({ message: "Not found" });
|
||||
res.json(rows[0]);
|
||||
});
|
||||
|
||||
app.get("/api/pids", async (req, res) => {
|
||||
const [rows] = await db.query("SELECT DISTINCT pid FROM pid_logs ORDER BY pid");
|
||||
res.json(rows.map((row) => row.pid));
|
||||
});
|
||||
|
||||
// === Auto crawl logs from internal directory ===
|
||||
async function fetchLogFile(url) {
|
||||
const res = await axios.get(url);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
async function crawlLogs() {
|
||||
const BASE_URL = "http://172.16.5.7:8080";
|
||||
console.log(`[${new Date().toISOString()}] Start crawl ${BASE_URL}`);
|
||||
const res = await axios.get(BASE_URL);
|
||||
const $ = cheerio.load(res.data);
|
||||
const links = $("a")
|
||||
.map((_, el) => $(el).attr("href"))
|
||||
.get()
|
||||
.filter((href) => href.endsWith(".log"));
|
||||
|
||||
const splitBlocks = (logContent) =>
|
||||
logContent
|
||||
.split(/(?=^.*?[#>]\s*sh(ow)?\s+inv(entory)?)/gim)
|
||||
.map((b) => b?.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const link of links) {
|
||||
const fileUrl = `${BASE_URL}/${link}`;
|
||||
const filename = link.replace(/\//g, "_");
|
||||
// const [exists] = await db.query(
|
||||
// "SELECT COUNT(*) as count FROM pid_logs WHERE log LIKE ?",
|
||||
// [`%${filename}%`]
|
||||
// );
|
||||
// if (exists[0].count > 0) {
|
||||
// console.log(`🟡 Skipped: ${filename}`);
|
||||
// continue;
|
||||
// }
|
||||
|
||||
try {
|
||||
const log = await fetchLogFile(fileUrl);
|
||||
const blocks = splitBlocks(log);
|
||||
|
||||
for (const block of blocks) {
|
||||
const { pid, version } = extractPIDVersion(block);
|
||||
const commands = extractCommands(block);
|
||||
const errors = extractLogging(block);
|
||||
|
||||
if (pid === "UNKNOWN" || version === "UNKNOWN") continue;
|
||||
|
||||
const [existing] = await db.query(
|
||||
"SELECT * FROM pid_logs WHERE pid = ? AND version = ?",
|
||||
[pid, version]
|
||||
);
|
||||
|
||||
// Nếu CHƯA CÓ -> INSERT mới toàn bộ
|
||||
if (!existing[0]) {
|
||||
const relevantLog = extractRelevantLog(block, commands);
|
||||
|
||||
await db.query(
|
||||
`INSERT INTO pid_logs (pid, version, log, commands, errors) VALUES (?, ?, ?, ?, ?)`,
|
||||
[
|
||||
pid,
|
||||
version,
|
||||
`[${filename}]\n${relevantLog}`,
|
||||
JSON.stringify(commands),
|
||||
errors.join('\n'),
|
||||
]
|
||||
);
|
||||
|
||||
console.log(`✅ Added ${pid} ${version} (${filename})`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Nếu ĐÃ CÓ -> Kiểm tra và UPDATE nếu cần
|
||||
const oldCommands = existing[0].commands;
|
||||
|
||||
const isSameCommands = Object.keys(commands).every(
|
||||
(key) => commands[key] === oldCommands[key]
|
||||
);
|
||||
|
||||
if (isSameCommands) {
|
||||
console.log(`🟡 Skipped existing PID+Version: ${pid} ${version}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const newCommands = {};
|
||||
for (const key of Object.keys(commands)) {
|
||||
if (commands[key] === true && oldCommands[key] === false) {
|
||||
newCommands[key] = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(newCommands).length === 0) {
|
||||
console.log(`🟠 No new true commands → skip ${pid} ${version}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const newRelevantLog = extractRelevantLog(block, newCommands);
|
||||
const updatedCommands = { ...oldCommands, ...newCommands };
|
||||
const updatedLog = `${existing[0].log}\n[${filename}]\n${newRelevantLog}`;
|
||||
|
||||
await db.query(
|
||||
`UPDATE pid_logs SET log = ?, commands = ?, errors = ? WHERE pid = ? AND version = ?`,
|
||||
[
|
||||
updatedLog,
|
||||
JSON.stringify(updatedCommands),
|
||||
`${existing[0].errors || ''}\n${errors.join('\n')}`.trim(),
|
||||
pid,
|
||||
version,
|
||||
]
|
||||
);
|
||||
|
||||
console.log(`🟢 Updated with new command(s) for ${pid} ${version}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`❌ Error ${filename}: ${err.message} ${err.stack}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// === Run crawl on start ===
|
||||
// crawlLogs();
|
||||
|
||||
app.listen(PORT, () =>
|
||||
console.log(`Server running at http://localhost:${PORT}`)
|
||||
);
|
||||
Loading…
Reference in New Issue