import fs from 'node:fs' import { textfsmResults } from './../ultils/templates/index.js' import net from 'node:net' import { appendLog, applyRules, classifyLog, cleanData, detectScenarioByModel, escapeHtml, isValidJson, LogStreamBuffer, mapErrorsToRows, mapToLineFormat, sendMessageToMail, sleep, TestSession, updateNoteToERP, } from '../ultils/helper.js' import Scenario from '#models/scenario' import path from 'node:path' import axios from 'axios' import redis from '@adonisjs/redis/services/main' import Line from '#models/line' import { ErrorRow, TestResult } from '../ultils/types.js' import moment from 'moment' import momentTZ from 'moment-timezone' type Inventory = { pid: string vid: string sn: string licenseLevel: string licenseType: string nextLicenseLevel: string } interface LineConfig { id: number port: number lineNumber: number ip: string stationId: number stationName: string stationIp: string apcName?: string outlet: number output: string status: string baud: number openCLI: boolean userEmailOpenCLI: string userOpenCLI: string inventory: any latestScenario?: { name: string time: number detectAI?: { status: string[] issue: string[] summary: string } } data: { command: string output: string textfsm: string }[] commands: string[] // history: string } /** HISTORY * PID * SN * VID * Timestamp * Scenario */ interface HistoryItem { pid: string vid: string sn: string scenario: string id: number number: number stationId: number timestamp?: number } interface User { userEmail: string userName: string } interface DataDPELP { line: number pid: any vid: any sn: any ios: string mac: string license: any issues: string[] summary: string } export default class LineConnection { public client: net.Socket public config: LineConfig public readonly socketIO: any private outputBuffer: string private isRunningScript: boolean private connecting: boolean private waitingScenario: boolean private outputInventory: string private outputScenario: string private bufferLog: LogStreamBuffer public dataDPELP: DataDPELP | string private listScenarios: number[] public handleClearLine: () => void private session: TestSession constructor(config: LineConfig, socketIO: any, handleClearLine: () => void) { this.config = config this.socketIO = socketIO this.client = new net.Socket() this.outputBuffer = '' this.isRunningScript = false this.connecting = false this.waitingScenario = false this.outputInventory = '' this.outputScenario = '' this.bufferLog = new LogStreamBuffer() this.dataDPELP = { line: this.config.lineNumber, pid: '', vid: '', sn: '', ios: '', mac: '', license: [], issues: ['No data'], summary: '', } this.listScenarios = [] this.session = new TestSession() this.handleClearLine = handleClearLine } connect(timeoutMs = 5000) { return new Promise((resolve, reject) => { const { ip, port, lineNumber, id, stationId } = this.config let resolvedOrRejected = false // Set timeout this.client.setTimeout(timeoutMs) console.log(`🔌 Connecting to line ${lineNumber} (${ip}:${port})...`) this.client.connect(port, ip, () => { if (resolvedOrRejected) return resolvedOrRejected = true console.log(`[${Date.now()}] ✅ Connected to line ${lineNumber} (${ip}:${port})`) this.connecting = true setTimeout(() => { this.config.status = 'connected' // this.retryConnect = 0 this.connecting = false this.socketIO.emit('line_connected', { stationId, lineId: id, lineNumber, status: 'connected', }) this.checkLog() resolve() }, 1000) }) this.client.on('data', (data) => { let message = this.connecting ? cleanData(data.toString()) : data.toString() const lines = this.bufferLog.push(data) lines.forEach(this.handleLogLine) let rawData = '' if (this.isRunningScript) { this.waitingScenario = true this.outputBuffer += message this.outputScenario += message if (!this.config.inventory) this.outputInventory = this.outputInventory.slice(-3000) + message } if (data.toString().includes('More') || data.toString().includes('MORE')) this.writeCommand(' ') // let output = cleanData(message) // console.log(`📨 [${this.config.port}] ${message}`) // Handle netOutput with backspace support for (const char of message) { if (char === '\x7F' || char === '\x08') { this.config.output = this.config.output.slice(0, -1) // message = message.slice(0, -1) } else { rawData += char } } this.config.output += cleanData(rawData) this.config.output = this.config.output.slice(-15000) this.socketIO.emit('line_output', { stationId, lineId: id, data: message, commands: this.config.commands, }) if (!this.config.inventory) { setTimeout(() => { this.getInventory() }, 5000) } appendLog( cleanData(message), this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) }) this.client.on('error', (err) => { if (resolvedOrRejected) return resolvedOrRejected = true console.error(`❌ Error line ${lineNumber}:`, err.message) this.config.output += '\r\n' + err.message + '\r\n' this.socketIO.emit('line_error', { stationId, lineId: id, error: '\r\n' + err.message + '\r\n', }) resolve() }) this.client.on('close', async () => { console.log(`[${Date.now()}] 🔌 Line ${lineNumber} disconnected`) this.config.status = 'disconnected' // this.config.inventory = undefined this.socketIO.emit('line_disconnected', { stationId, lineId: id, lineNumber, status: 'disconnected', }) // if (this.retryConnect <= 5) { // await this.sleep(5000) // console.log(`Retry connect line [${this.config.lineNumber}] times`, this.retryConnect) // this.retryConnect += 1 // await this.reconnect() // } else { // this.retryConnect = 0 // } }) this.client.on('timeout', () => { if (resolvedOrRejected) return resolvedOrRejected = true const message = '\r\nConnection timeout!!\r\n' this.config.output += message this.socketIO.emit('line_output', { stationId, lineId: id, data: message, }) appendLog( cleanData(message), this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) console.log(`⏳ Connection timeout line ${lineNumber}`) this.client.destroy() resolve() // reject(new Error('Connection timeout')) }) }) } private sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) } async writeCommand(cmd: string | Buffer, userName = '') { if (this.client.destroyed) { console.log(`⚠️ Cannot send, line ${this.config.lineNumber} is closed`) this.disconnect() // this.disconnect() // await sleep(2000) // await this.connect() return } // console.log( // `Write command "${cmd.toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}" to line ${this.config.lineNumber} of ${this.config.stationName}` // ) this.client.write(cmd) } async disconnect() { try { console.log('[DISCONNECT] Line', this.config.lineNumber) // this.handleClearLine() this.client.destroy() this.config.status = 'disconnected' this.socketIO.emit('line_disconnected', { ...this.config, status: 'disconnected', }) console.log(`🔻 Closed connection to line ${this.config.lineNumber}`) } catch (e) { console.error('Error closing line:', e) } } async runScript(script: Scenario, userName: string) { if (!this.client || this.client.destroyed) { console.log('Not connected') this.isRunningScript = false this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) this.outputBuffer = '' return } if (this.isRunningScript) { console.log('Script already running') return } console.log( `Run scenario "${script?.title}" to line ${this.config.lineNumber} of ${this.config.stationName}` ) this.isRunningScript = true this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: script?.title, }) if (script?.send_result || script?.sendResult) { this.dataDPELP = '' this.config.inventory = '' } if (script?.isReboot) { await sleep(10000) for (let index = 0; index < 30; index++) { await sleep(1000) this.breakSpam() } } const now = Date.now() this.outputScenario += `\n\n---start-scenarios---${now}---${userName}---${script?.title}---\n---scenario---${script?.title}---${now}---\n` appendLog( `\n\n---start-scenarios---${now}---${userName}---${script?.title}---\n---scenario---${script?.title}---${now}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) this.config.latestScenario = { name: script?.title, time: now, } const steps = typeof script?.body === 'string' ? JSON.parse(script?.body) : [] let stepIndex = 0 return new Promise((resolve, reject) => { const timeoutTimer = setTimeout( () => { this.isRunningScript = false this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) this.outputBuffer = '' this.outputScenario = '' this.config.output += 'Timeout run scenario' this.dataDPELP = { line: this.config.lineNumber, pid: '', vid: '', sn: '', ios: '', mac: '', license: [], issues: ['No data'], summary: '', } this.socketIO.emit('line_output', { stationId: this.config.stationId, lineId: this.config.id, data: 'Timeout run scenario', }) this.outputScenario += `\n---end-scenarios---${now}---${userName}---\n` appendLog( `\n---end-scenarios---${now}---${userName}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) // reject(new Error('Script timeout')) }, script.timeout ? Number(script.timeout) * 1000 : 300000 ) const runStep = async (index: number) => { if (index >= steps.length) { if (this.waitingScenario) { this.waitingScenario = false setTimeout(() => { runStep(index) }, 5000) return } else clearTimeout(timeoutTimer) this.isRunningScript = false this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) const logScenarios = this.outputScenario const data = textfsmResults(logScenarios, '') let pid = this.config.inventory?.pid || '' try { data.forEach((item) => { if (item?.textfsm && isValidJson(item?.textfsm)) { if ( ['show inventory', 'sh inventory', 'show inv', 'sh inv'].includes(item.command) ) { const dataInventory = JSON.parse(item.textfsm)[0] this.config.inventory = dataInventory pid = dataInventory?.pid || '' this.addHistory(this.config.stationId, this.config.id, { id: this.config.id, number: this.config.lineNumber, stationId: this.config.stationId, pid: dataInventory?.pid, sn: dataInventory?.sn, vid: dataInventory?.vid, scenario: script?.title, timestamp: Date.now(), }) } item.textfsm = JSON.parse(item.textfsm) } }) const scenario = await detectScenarioByModel(pid, this.listScenarios) console.log(pid, scenario?.title, this.listScenarios) if ( scenario && scenario.id !== script.id && scenario.title.includes('DPELP') && script.title.includes('DPELP') ) { this.listScenarios.push(scenario.id) // this.outputScenario = '' this.runScript(scenario, userName) // this.socketIO.emit('confirm_scenario', { // scenario: scenario, // id: this.config.id, // }) resolve(true) return } const detectLog = await this.detectLogWithAI(logScenarios) const result = mapToLineFormat({ lineNumber: this.config.lineNumber, inventory: this.config.inventory, latestScenario: { detectAI: detectLog, }, data, }) // if (script?.send_result || script?.sendResult) { this.dataDPELP = result console.log( `DPELP DATA line ${this.config.lineNumber} of ${this.config.stationName}:`, this.dataDPELP ) // } if (this.config.latestScenario) this.config.latestScenario = { ...this.config.latestScenario, detectAI: detectLog } this.config.data = data this.socketIO.emit('data_textfsm', { stationId: this.config.stationId, lineId: this.config.id, data, inventory: this.config.inventory || null, latestScenario: this.config.latestScenario || null, }) if (result.sn) { this.updateNote(result.sn, result) } } catch (error) { console.log(error) } this.outputBuffer = '' this.outputScenario += `\n---end-scenarios---${now}---${userName}---\n` appendLog( `\n---end-scenarios---${now}---${userName}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) this.listScenarios = [] resolve(true) return } const step = steps[index] let repeatCount = Number(step.repeat) || 1 const delay = step?.delay ? Number(step?.delay) * 1000 : 1000 const sendCommand = async () => { if (repeatCount <= 0) { // Done → next step stepIndex++ return runStep(stepIndex) } if (typeof step.send !== 'undefined') { this.outputScenario += `\n---send-command---"${(step?.send ?? '[ENTER]').toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}"---${now}---\n` appendLog( `\n---send-command---"${(step?.send ?? '[ENTER]').toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}"---${now}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) this.writeCommand((step?.send || '') + '\r\n') } repeatCount-- setTimeout(() => sendCommand(), delay) } // Nếu expect rỗng → gửi ngay if (!step?.expect || step?.expect.trim() === '') { setTimeout(() => sendCommand(), delay) return } // while (this.outputBuffer) { // await sleep(200) // if (this.outputBuffer.includes(step.expect)) { // this.outputBuffer = '' // setTimeout(() => sendCommand(), delay) // } // } const matched = await this.waitForExpect( step.expect.trim(), script?.timeout ? Number(script?.timeout) * 1000 : 60000 ) if (matched) setTimeout(() => sendCommand(), delay) } runStep(stepIndex) }) } public async reconnect(): Promise { try { this.disconnect() this.client = new net.Socket() await this.sleep(1000) await this.connect() return true } catch (err: any) { return false } } userOpenCLI(user: User) { this.config.openCLI = true this.config.userEmailOpenCLI = user.userEmail this.config.userOpenCLI = user.userName this.socketIO.emit('user_open_cli', { stationId: this.config.stationId, lineId: this.config.id, userEmailOpenCLI: user.userEmail, userOpenCLI: user.userName, }) appendLog( `\n-------${user.userName}-------\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) } userCloseCLI() { this.config.openCLI = false this.config.userEmailOpenCLI = '' this.config.userOpenCLI = '' this.socketIO.emit('user_close_cli', { stationId: this.config.stationId, lineId: this.config.id, userEmailOpenCLI: '', }) } clearCLI() { this.config.output = '' this.socketIO.emit('user_clear_terminal', { stationId: this.config.stationId, lineId: this.config.id, }) setTimeout(() => this.writeCommand('\r\n'), 100) } waitForExpect = async (expect: string, timeout = 60000) => { const start = Date.now() // console.log('[EXPECT]', expect, timeout) while (Date.now() - start < timeout) { if (this.outputBuffer.includes(expect)) { this.outputBuffer = '' return true } await sleep(200) } return false } getInventory = () => { const data = textfsmResults(this.outputInventory, 'show inventory') try { data.forEach((item) => { if (item?.textfsm && isValidJson(item?.textfsm)) { if (['show inventory', 'sh inventory', 'show inv', 'sh inv'].includes(item.command)) { const dataInventory = JSON.parse(item.textfsm)[0] this.config.inventory = dataInventory } item.textfsm = JSON.parse(item.textfsm) } }) if (this.config.inventory) { this.config.data = data this.socketIO.emit('data_textfsm', { stationId: this.config.stationId, lineId: this.config.id, data, inventory: this.config.inventory || null, latestScenario: this.config.latestScenario || null, }) this.outputInventory = '' } } catch (error) { console.log(error) } } // Gửi nhiều ký tự ESC để vào ROMMON breakSpam() { console.log('SPAM Break to line:', this.config.lineNumber) let count = 0 const escInterval = setInterval(() => { if (count >= 10) { clearInterval(escInterval) return } this.client.write(Buffer.from([0xff, 0xf3])) // Ctrl + Break count++ }, 1) } async setBaud(baud: number) { this.writeCommand('enable\r\n') await sleep(500) this.writeCommand('configure terminal\r\n') await sleep(500) this.writeCommand('line console 0\r\n') await sleep(500) this.writeCommand(`speed ${baud.toString()}\r\n`) await sleep(500) this.writeCommand('end\r\n') await sleep(500) this.writeCommand('write memory\r\n') this.writeCommand('\r\n') } async getLog(date: string) { const logDir = path.join('storage', 'system_logs') const logFile = path .join( logDir, `${date}-AUTO-Session.${this.config.stationName}-${this.config.stationId}-${this.config.stationIp}-${this.config.lineNumber}.log` ) .replaceAll(' ', '_') if (!fs.existsSync(logDir) || !fs.existsSync(logFile)) { return '' } return await fs.promises.readFile(logFile, 'utf8') } async detectLogWithAI(log: string) { try { const payload = { model: 'gpt-4o-mini', max_tokens: 1000, messages: [ { role: 'user', content: `You are a network hardware tester. Your task is to analyze router/switch logs to determine whether the device meets hardware standards for reselling. Focus ONLY on hardware-related problems or abnormal warnings. Software or configuration issues (e.g., port up/down, admin down, invalid commands, CLI errors, licensing messages) should be ignored unless they indicate hardware failure. OUTPUT FORMAT (must follow exactly): { "issue": [ "problem 1", "problem 2", ... ], "summary": "short summary under 30 words" } RULES: - Summaries must be in English. - Each issue must be one short line. - If the log contains no hardware issues, output: { "issue": ["No issues detected."], "summary": "No hardware issues found." } - Keep responses concise, readable, and strictly in JSON format. - Do NOT add explanations outside the JSON. - Your job is to detect hardware faults, missing components, overheating, failing modules, PSU issues, sensor anomalies, SIM/card missing, modem errors, transceiver issues, POST/diagnostics failures, etc. The log to analyze will be provided after this prompt. Here is the log: ${log} `, }, ], } const remoteUrl = process.env.ERP_URL || 'https://stage.nswteam.net' const remoteResp = await axios.post( remoteUrl + '/api/transferPostData', { urlAPI: '/api/open-ai-sfp/model-image-info', data: payload, }, { headers: { Authorization: 'Bearer ' + process.env.ERP_TOKEN, }, } ) return remoteResp.data?.Status === 'OK' ? remoteResp.data?.data : '' } catch (error: any) { console.log('[ERROR] Detect log from AI', error) } return '' } async addHistory(stationId: number, lineId: number, item: HistoryItem) { if (!item.pid || !item.sn) return const key = `station:${stationId}:line:${lineId}:history` const now = Date.now() const newItem = JSON.stringify({ ...item, timestamp: now, }) // Lấy phần tử cuối const lastItems = await redis.zrevrange(key, 0, 0) if (lastItems.length > 0) { const last = JSON.parse(lastItems[0]) if (last.pid === item.pid && last.sn === item.sn) { return false // không thay đổi } } const line = await Line.find(lineId) if (line) { const listHistory = line.history ? JSON.parse(line.history) : [] listHistory.unshift(newItem) line.history = JSON.stringify(listHistory) await line.save() } // Add vào ZSET await redis.zadd(key, now, newItem) // Tự động xóa item > 96h const expireTime = now - 96 * 60 * 60 * 1000 await redis.zremrangebyscore(key, 0, expireTime) return true } async getHistory(stationId: number, lineId: number) { const key = `station:${stationId}:line:${lineId}:history` const items = await redis.zrange(key, 0, -1) return items.map((i) => JSON.parse(i)) } handleLogLine = (line: string) => { try { const parsed = classifyLog(line) this.session.applyParsedLog(parsed) } catch (error) { console.log('handleLogLine', error) } } checkLog() { const interval = setInterval(async () => { try { if (this.config.status !== 'connected') { clearInterval(interval) this.session.clear() this.bufferLog.clear() return } const result = this.session.finalize() if (result.errors.length === 0) { this.session.clear() this.bufferLog.clear() return } const detectLog = await this.detectLogWithAI(this.bufferLog.allBuffer) // console.log(detectLog) const tableHTML = this.buildEmailContent(result, detectLog) await sendMessageToMail( 'andrew.ng@apactech.io', `[ATC] - [${this.config.stationName} - Line: ${this.config.lineNumber}] - Raw log issue`, tableHTML + `${`

Logs:

${this.bufferLog.allBuffer}
`}`, ['ips@ipsupply.com.au', 'kay@ipsupply.com.au', 'joseph@apactech.io'] ) this.session.clear() this.bufferLog.clear() } catch (err: any) { console.error('Error checking log:', err) } }, 300000) } renderErrorTable(rows: ErrorRow[]): string { if (!rows.length) { return `

No errors detected

` } const header = ` Level Rule Message Log Evidence ` const body = rows .map( (r) => ` ${r.level} ${r.rule} ${r.message}
${escapeHtml(r.log.trim())}
` ) .join('') return ` ${header} ${body}
` } renderAIDetectTable(row: any): string { return `
Summary Issues
${row.summary || ''} ${row.issues?.length ? `- ` + row.issues.join(`
- `) : '- No issues detected.'}
` } buildEmailContent(result: TestResult, value: any): string { const rows = mapErrorsToRows(result.errors) const table = this.renderErrorTable(rows) // const tableAI = this.renderAIDetectTable(value) return `

Cisco Device Log Result

Line: ${this.config.lineNumber} - Station: ${this.config.stationName}

Summary: ${result.summary}. ${value.summary}


${table}
` } async updateNote(sn: string, data: DataDPELP) { const licenses = Array.isArray(data.license) ? [...new Set(data.license)] : data.license ? [data.license] : [] const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const dataFormat = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm') const note = `-------[ATC]-[${dataFormat}]-------\nLicense: ${licenses.join(', ')}\nSummary: ${data?.summary || ''}\nIssues:\n${data.issues?.length ? `- ` + data.issues.join(`\n- `) : ''}\n\n` await updateNoteToERP(sn, note) } }