import fs from 'node:fs' import { textfsmResults } from './../ultils/templates/index.js' import net from 'node:net' import { appendLog, buildBody, classifyLog, cleanData, detectConfigRamByModel, detectScenarioByModel, escapeHtml, isRamSufficient, isValidJson, LogStreamBuffer, mapErrorsToRows, mapToLineFormat, normalizeInterface, parseLicenseReport, sendMessageToMail, sleep, TestSession, updateNoteToERP, } from '../ultils/helper.js' import Scenario from '#models/scenario' import path, { join } from 'node:path' import axios from 'axios' import redis from '@adonisjs/redis/services/main' import Line from '#models/line' import { ErrorRow, TestResult } from '../ultils/types.js' import momentTZ from 'moment-timezone' import { PhysicalPortTest } from './physical_test_service.js' import Station from '#models/station' import IosLicenseController from '#controllers/ios_license_controller' import ConfigRam from '#models/config_ram' type Inventory = { pid: string vid: string sn: string licenseLevel: string licenseType: string nextLicenseLevel: string } interface LineConfig { id: number port: number lineNumber: number ip: string stationId: number stationName: string stationIp: string apcName?: string outlet: number output: string status: string baud: number openCLI: boolean userEmailOpenCLI: string userOpenCLI: string inventory: any latestScenario?: { name: string time: number detectAI?: { status: string[] issue: string[] summary: string } } data: { command: string output: string textfsm: string }[] ports: string[] runningScenario: string runningPhysical: boolean // history: string } /** HISTORY * PID * SN * VID * Timestamp * Scenario */ interface HistoryItem { pid: string vid: string sn: string scenario: string id: number number: number stationId: number timestamp?: number } interface User { userEmail: string userName: string } interface DataDPELP { line: number pid: any vid: any sn: any ios: string mac: string license: any issues: string[] summary: string } export default class LineConnection { public client: net.Socket public config: LineConfig public readonly socketIO: any private outputBuffer: string private connecting: boolean private waitingScenario: boolean private outputInventory: string private outputScenario: string private bufferLog: LogStreamBuffer public dataDPELP: DataDPELP | string private listScenarios: number[] public handleClearLine: () => void private session: TestSession public physicalTest: PhysicalPortTest private outputPhysicalTest: string private outputLoadIosLicense: string | boolean private listDeviceIos: string[] private debounceTimer: NodeJS.Timeout | null = null private testingPortPoE: boolean constructor(config: LineConfig, socketIO: any, handleClearLine: () => void) { this.config = config this.socketIO = socketIO this.client = new net.Socket() this.outputBuffer = '' this.connecting = false this.waitingScenario = false this.outputInventory = '' this.outputScenario = '' this.bufferLog = new LogStreamBuffer() this.dataDPELP = { line: this.config.lineNumber, pid: '', vid: '', sn: '', ios: '', mac: '', license: [], issues: ['No data'], summary: '', } this.listScenarios = [] this.session = new TestSession() this.handleClearLine = handleClearLine this.physicalTest = new PhysicalPortTest([]) this.outputPhysicalTest = '' this.outputLoadIosLicense = '' this.listDeviceIos = [] this.debounceTimer = null this.testingPortPoE = false } /** * Connect to line with socket */ connect(timeoutMs = 5000) { return new Promise((resolve, reject) => { const { ip, port, lineNumber, id, stationId } = this.config let resolvedOrRejected = false // Set timeout this.client.setTimeout(timeoutMs) console.log(`🔌 Connecting to line ${lineNumber} (${ip}:${port})...`) this.client.connect(port, ip, () => { if (resolvedOrRejected) return resolvedOrRejected = true console.log(`[${Date.now()}] ✅ Connected to line ${lineNumber} (${ip}:${port})`) this.connecting = true setTimeout(() => { this.config.status = 'connected' // this.retryConnect = 0 this.connecting = false this.socketIO.emit('line_connected', { stationId, lineId: id, lineNumber, status: 'connected', }) this.checkLog() resolve() }, 1000) }) this.client.on('data', (data) => { let message = this.connecting ? cleanData(data.toString()) : data.toString() const lines = this.bufferLog.push(data) lines.forEach(this.handleLogLine) let rawData = '' if (this.config.runningScenario) { this.waitingScenario = true this.outputBuffer += message this.outputScenario += message if (!this.config.inventory) this.outputInventory = this.outputInventory.slice(-3000) + message } if (this.outputLoadIosLicense) { if (this.outputLoadIosLicense === true) this.outputLoadIosLicense = '' this.outputLoadIosLicense += cleanData(data.toString()) } if (this.config.runningPhysical) { this.outputPhysicalTest += message if (this.debounceTimer) clearTimeout(this.debounceTimer) if (this.testingPortPoE) this.debounceTimer = setTimeout(() => { this.flushLogBuffer() }, 1000) // 1s debounce } if (data.toString().includes('More') || data.toString().includes('MORE')) this.writeCommand(' ') // let output = cleanData(message) // console.log(`📨 [${this.config.port}] ${message}`) // Handle netOutput with backspace support for (const char of message) { if (char === '\x7F' || char === '\x08') { this.config.output = this.config.output.slice(0, -1) // message = message.slice(0, -1) } else { rawData += char } } this.config.output += cleanData(rawData) this.config.output = this.config.output.slice(-15000) this.socketIO.emit('line_output', { stationId, lineId: id, data: message, ports: this.config.ports, }) setTimeout(() => { if (!this.config.inventory) { this.getInventory() } }, 5000) appendLog( cleanData(message), this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) }) this.client.on('error', (err) => { if (resolvedOrRejected) return resolvedOrRejected = true console.error(`❌ Error line ${lineNumber}:`, err.message) this.config.output += '\r\n' + err.message + '\r\n' this.socketIO.emit('line_error', { stationId, lineId: id, error: '\r\n' + err.message + '\r\n', }) this.endTesting() resolve() }) this.client.on('close', async () => { console.log(`[${Date.now()}] 🔌 Line ${lineNumber} disconnected`) this.config.status = 'disconnected' // this.config.inventory = undefined this.socketIO.emit('line_disconnected', { stationId, lineId: id, lineNumber, status: 'disconnected', }) // if (this.retryConnect <= 5) { // await this.sleep(5000) // console.log(`Retry connect line [${this.config.lineNumber}] times`, this.retryConnect) // this.retryConnect += 1 // await this.reconnect() // } else { // this.retryConnect = 0 // } this.endTesting() }) this.client.on('timeout', () => { if (resolvedOrRejected) return resolvedOrRejected = true const message = '\r\nConnection timeout!!\r\n' this.config.output += message this.socketIO.emit('line_output', { stationId, lineId: id, data: message, }) appendLog( cleanData(message), this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) console.log(`⏳ Connection timeout line ${lineNumber}`) this.client.destroy() resolve() // reject(new Error('Connection timeout')) }) }) } /** * Waiting with millisecond */ private sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) } /** * Write a command with socket.write */ async writeCommand(cmd: string | Buffer, userName = '') { if (this.client.destroyed) { console.log(`⚠️ Cannot send, line ${this.config.lineNumber} is closed`) this.disconnect() // this.disconnect() // await sleep(2000) // await this.connect() return } // console.log( // `Write command "${cmd.toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}" to line ${this.config.lineNumber} of ${this.config.stationName}` // ) this.client.write(cmd) } /** * Disconnect socket with line */ async disconnect() { try { console.log('[DISCONNECT] Line', this.config.lineNumber) // this.handleClearLine() this.client.destroy() this.config.status = 'disconnected' this.socketIO.emit('line_disconnected', { ...this.config, status: 'disconnected', }) console.log(`🔻 Closed connection to line ${this.config.lineNumber}`) } catch (e) { console.error('Error closing line:', e) } } /** * Run a scenario as DPELP, Breaking password, load ios,... */ async runScript(script: Scenario, userName: string) { if (!this.client || this.client.destroyed) { console.log('Not connected') this.config.runningScenario = '' this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) this.outputBuffer = '' return } if (this.config.runningScenario || this.config.runningPhysical) { console.log('Script already running') return } console.log( `Run scenario "${script?.title}" to line ${this.config.lineNumber} of ${this.config.stationName}` ) this.config.runningScenario = script?.title this.config.data = [] this.outputScenario = '' this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: script?.title, }) if (script?.send_result || script?.sendResult) { this.dataDPELP = '' this.config.inventory = '' } if (script?.isReboot) { await sleep(10000) for (let index = 0; index < 30; index++) { await sleep(1000) this.breakSpam() } } const now = Date.now() this.outputScenario += `\n\n---start-scenarios---${now}---${userName}---${script?.title}---\n---scenario---${script?.title}---${now}---\n` appendLog( `\n\n---start-scenarios---${now}---${userName}---${script?.title}---\n---scenario---${script?.title}---${now}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) this.config.latestScenario = { name: script?.title, time: now, } const steps = typeof script?.body === 'string' ? JSON.parse(script?.body) : [] let stepIndex = 0 // Create a timeout let timeoutTimer: NodeJS.Timeout | null = null const timeoutNumber = script.timeout ? Number(script.timeout) * 1000 : 300000 const onTimeout = () => { this.config.runningScenario = '' this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) this.outputBuffer = '' this.outputScenario = '' this.config.output += 'Timeout run scenario' this.dataDPELP = { line: this.config.lineNumber, pid: '', vid: '', sn: '', ios: '', mac: '', license: [], issues: ['No data'], summary: '', } this.socketIO.emit('line_output', { stationId: this.config.stationId, lineId: this.config.id, data: 'Timeout run scenario', }) this.outputScenario += `\n---end-scenarios---${now}---${userName}---\n` appendLog( `\n---end-scenarios---${now}---${userName}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) // reject(new Error('Script timeout')) } const resetTimeout = () => { // console.log('resetTimeout', timeoutNumber) // this.outputBuffer = '' if (timeoutTimer) clearTimeout(timeoutTimer) timeoutTimer = setTimeout(onTimeout, timeoutNumber) } return new Promise((resolve, reject) => { timeoutTimer = setTimeout(onTimeout, timeoutNumber) const runStep = async (index: number) => { if (index >= steps.length) { if (this.waitingScenario) { this.waitingScenario = false setTimeout(() => { runStep(index) }, 5000) return } else if (timeoutTimer) clearTimeout(timeoutTimer) this.outputScenario += `\n---end-scenarios---${now}---${userName}---\n` this.outputBuffer = '' this.config.runningScenario = '' this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) const logScenarios = this.outputScenario const data = textfsmResults(logScenarios, '') let pid = this.config.inventory?.pid || '' try { for (const item of data) { if (item?.textfsm && isValidJson(item?.textfsm)) { if ( ['show inventory', 'sh inventory', 'show inv', 'sh inv'].includes(item.command) ) { const dataInventory = JSON.parse(item.textfsm)[0] this.config.inventory = this.config.inventory ? { ...this.config.inventory, ...dataInventory } : dataInventory pid = dataInventory?.pid || '' this.addHistory(this.config.stationId, this.config.id, { id: this.config.id, number: this.config.lineNumber, stationId: this.config.stationId, pid: dataInventory?.pid, sn: dataInventory?.sn, vid: dataInventory?.vid, scenario: script?.title, timestamp: Date.now(), }) } if (['show version', 'sh version', 'show ver', 'sh ver'].includes(item.command)) { const dataVer = JSON.parse(item.textfsm)[0] this.config.inventory = this.config.inventory ? { ...this.config.inventory, ...dataVer } : dataVer if (pid && (dataVer?.MEMORY || dataVer?.USB_FLASH)) { await this.checkConfigRam( dataVer?.MEMORY || '', dataVer?.USB_FLASH || '', pid, cleanData(item.output) ) } } if ( item.command?.trim()?.includes('show env') || item.command?.trim()?.includes('sh env') ) { const dataEnv = await this.detectShowEnvWithAI(item.output) item.dataAI = dataEnv } item.textfsm = JSON.parse(item.textfsm) } } const scenario = await detectScenarioByModel(pid, this.listScenarios) console.log(pid, scenario?.title, this.listScenarios) if ( scenario && scenario.id !== script.id && scenario.title.includes('DPELP') && script.title.includes('DPELP') ) { this.listScenarios.push(scenario.id) // this.outputScenario = '' this.runScript(scenario, userName) // this.socketIO.emit('confirm_scenario', { // scenario: scenario, // id: this.config.id, // }) resolve(true) return } if (script?.send_result || script?.sendResult) { const detectLog = await this.detectLogWithAI(logScenarios) const result = mapToLineFormat({ lineNumber: this.config.lineNumber, inventory: this.config.inventory, latestScenario: { detectAI: detectLog, }, data, }) // if (script?.send_result || script?.sendResult) { this.dataDPELP = result console.log( `DPELP DATA line ${this.config.lineNumber} of ${this.config.stationName}:`, this.dataDPELP ) // } if (this.config.latestScenario) this.config.latestScenario = { ...this.config.latestScenario, detectAI: detectLog } if (result.sn) { this.updateNote(result.sn, result) } } this.config.data = data this.socketIO.emit('data_textfsm', { stationId: this.config.stationId, lineId: this.config.id, data, inventory: this.config.inventory || null, latestScenario: this.config.latestScenario || null, }) } catch (error) { console.log(error) } appendLog( `\n---end-scenarios---${now}---${userName}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) this.listScenarios = [] resolve(true) return } else resetTimeout() const step = steps[index] let repeatCount = Number(step.repeat) || 1 const delay = step?.delay ? Number(step?.delay) * 1000 : 1000 const sendCommand = async () => { // if (repeatCount <= 0) { // // Done → next step // stepIndex++ // return runStep(stepIndex) // } if (typeof step.send !== 'undefined') { console.log(Date.now() - now, (step?.send ?? '[ENTER]').toString()) this.outputScenario += `\n---send-command---"${(step?.send ?? '[ENTER]').toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}"---${now}---\n` appendLog( `\n---send-command---"${(step?.send ?? '[ENTER]').toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}"---${now}---\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) this.writeCommand((step?.send || '') + '\r\n') } repeatCount-- if (repeatCount <= 0) { // Done → next step stepIndex++ return runStep(stepIndex) } else setTimeout(() => sendCommand(), delay) } // Nếu expect rỗng → gửi ngay if (!step?.expect || step?.expect.trim() === '') { setTimeout(() => sendCommand(), delay) return } // while (this.outputBuffer) { // await sleep(200) // if (this.outputBuffer.includes(step.expect)) { // this.outputBuffer = '' // setTimeout(() => sendCommand(), delay) // } // } const matched = await this.waitForExpect( step.expect.trim(), script?.timeout ? Number(script?.timeout) * 1000 : 60000 ) if (matched) setTimeout(() => sendCommand(), delay) } runStep(stepIndex) }) } /** * Reconnect socket with line */ public async reconnect(): Promise { try { this.disconnect() this.client = new net.Socket() await this.sleep(1000) await this.connect() return true } catch (err: any) { return false } } /** * User open CLI from front-end */ userOpenCLI(user: User) { this.config.openCLI = true this.config.userEmailOpenCLI = user.userEmail this.config.userOpenCLI = user.userName this.socketIO.emit('user_open_cli', { stationId: this.config.stationId, lineId: this.config.id, userEmailOpenCLI: user.userEmail, userOpenCLI: user.userName, }) appendLog( `\n-------${user.userName}-------\n`, this.config.stationId, this.config.stationName, this.config.stationIp, this.config.lineNumber ) } /** * User close CLI from front-end */ userCloseCLI() { this.config.openCLI = false this.config.userEmailOpenCLI = '' this.config.userOpenCLI = '' this.socketIO.emit('user_close_cli', { stationId: this.config.stationId, lineId: this.config.id, userEmailOpenCLI: '', }) } /** * Clear output buffer */ clearCLI() { this.config.output = '' this.socketIO.emit('user_clear_terminal', { stationId: this.config.stationId, lineId: this.config.id, }) setTimeout(() => this.writeCommand('\r\n'), 100) } /** * Waiting for a expect with until catch it from output */ waitForExpect = async (expect: string, timeout = 60000) => { const start = Date.now() // console.log('[EXPECT]', expect, timeout) while (Date.now() - start < timeout) { if (this.outputBuffer.includes(expect)) { this.outputBuffer = '' return true } await sleep(200) } return false } /** * Detect inventory data from output */ getInventory = () => { const data = textfsmResults(this.outputInventory, 'show inventory') try { data.forEach((item) => { if (item?.textfsm && isValidJson(item?.textfsm)) { if (['show inventory', 'sh inventory', 'show inv', 'sh inv'].includes(item.command)) { const dataInventory = JSON.parse(item.textfsm)[0] this.config.inventory = this.config.inventory ? { ...this.config.inventory, ...dataInventory } : dataInventory } item.textfsm = JSON.parse(item.textfsm) } }) if (this.config.inventory) { this.config.data = data this.socketIO.emit('data_textfsm', { stationId: this.config.stationId, lineId: this.config.id, data, inventory: this.config.inventory || null, latestScenario: this.config.latestScenario || null, }) this.outputInventory = '' } } catch (error) { console.log(error) } } /** * Gửi nhiều ký tự ESC để vào ROMMON */ breakSpam() { console.log('SPAM Break to line:', this.config.lineNumber) let count = 0 const escInterval = setInterval(() => { if (count >= 10) { clearInterval(escInterval) return } this.client.write(Buffer.from([0xff, 0xf3])) // Ctrl + Break count++ }, 1) } /** * Set Baud of line */ async setBaud(baud: number) { this.writeCommand('enable\r\n') await sleep(500) this.writeCommand('configure terminal\r\n') await sleep(500) this.writeCommand('line console 0\r\n') await sleep(500) this.writeCommand(`speed ${baud.toString()}\r\n`) await sleep(500) this.writeCommand('end\r\n') await sleep(500) this.writeCommand('write memory\r\n') this.writeCommand('\r\n') } /** * Get content's log of line with date */ async getLog(date: string) { const logDir = path.join('storage', 'system_logs') const logFile = path .join( logDir, `${date}-AUTO-Session.${this.config.stationName}-${this.config.stationId}-${this.config.stationIp}-${this.config.lineNumber}.log` ) .replaceAll(' ', '_') if (!fs.existsSync(logDir) || !fs.existsSync(logFile)) { return '' } return await fs.promises.readFile(logFile, 'utf8') } /** * Detect log by call api gpt, return summary and issues */ async detectLogWithAI(log: string) { try { const payload = { model: 'gpt-4o-mini', max_tokens: 1000, messages: [ { role: 'user', content: `You are a network hardware tester. Your task is to analyze router/switch logs to determine whether the device meets hardware standards for reselling. Focus ONLY on hardware-related problems or abnormal warnings. Software or configuration issues (e.g., port up/down, admin down, invalid commands, CLI errors, licensing messages) should be ignored unless they indicate hardware failure. OUTPUT FORMAT (must follow exactly): { "issue": [ "problem 1", "problem 2", ... ], "summary": "short summary under 30 words" } RULES: - Summaries must be in English. - Each issue must be one short line. - If the log contains no hardware issues, output: { "issue": ["No issues detected."], "summary": "No hardware issues found." } - Keep responses concise, readable, and strictly in JSON format. - Do NOT add explanations outside the JSON. - Your job is to detect hardware faults, missing components, overheating, failing modules, PSU issues, sensor anomalies, SIM/card missing, modem errors, transceiver issues, POST/diagnostics failures, etc. The log to analyze will be provided after this prompt. Here is the log: ${log} `, }, ], } const remoteUrl = process.env.ERP_URL || 'https://stage.nswteam.net' const remoteResp = await axios.post( remoteUrl + '/api/transferPostData', { urlAPI: '/api/open-ai-sfp/model-image-info', data: payload, }, { headers: { Authorization: 'Bearer ' + process.env.ERP_TOKEN, }, } ) return remoteResp.data?.Status === 'OK' ? remoteResp.data?.data : '' } catch (error: any) { console.log('[ERROR] Detect log from AI', error) } return '' } /** * Add cache to list history devices on this line */ async addHistory(stationId: number, lineId: number, item: HistoryItem) { if (!item.pid || !item.sn) return const key = `station:${stationId}:line:${lineId}:history` const now = Date.now() const newItem = JSON.stringify({ ...item, timestamp: now, }) // Lấy phần tử cuối const lastItems = await redis.zrevrange(key, 0, 0) if (lastItems.length > 0) { const last = JSON.parse(lastItems[0]) if (last.pid === item.pid && last.sn === item.sn) { return false // không thay đổi } } const line = await Line.find(lineId) if (line) { const listHistory = line.history ? JSON.parse(line.history) : [] listHistory.unshift(newItem) line.history = JSON.stringify(listHistory) await line.save() } // Add vào ZSET await redis.zadd(key, now, newItem) // Tự động xóa item > 96h const expireTime = now - 96 * 60 * 60 * 1000 await redis.zremrangebyscore(key, 0, expireTime) return true } /** * Get list history devices */ async getHistory(stationId: number, lineId: number) { const key = `station:${stationId}:line:${lineId}:history` const items = await redis.zrange(key, 0, -1) return items.map((i) => JSON.parse(i)) } /** * Handle raw log to regex error */ handleLogLine = (line: string) => { try { const parsed = classifyLog(line) this.session.applyParsedLog(parsed) } catch (error) { console.log('handleLogLine', error) } } /** * Check raw log was regex each 5 minutes, if has error will send email report */ checkLog() { const interval = setInterval(async () => { try { if (this.config.status !== 'connected') { clearInterval(interval) this.session.clear() this.bufferLog.clear() return } const result = this.session.finalize() if (result.errors.length === 0) { this.session.clear() this.bufferLog.clear() return } // const detectLog = await this.detectLogWithAI(this.bufferLog.allBuffer) // console.log(detectLog) const tableHTML = this.buildEmailContent(result) await sendMessageToMail( `[ATC] - [${this.config.stationName} - Line: ${this.config.lineNumber}] - Raw log issue`, tableHTML + `${`

Logs:

${this.bufferLog.allBuffer}
`}` ) this.session.clear() this.bufferLog.clear() } catch (err: any) { console.error('Error checking log:', err) } }, 300000) } /** * Render table to view error */ renderErrorTable(rows: ErrorRow[]): string { if (!rows.length) { return `

No errors detected

` } const header = ` Level Rule Message Log Evidence ` const body = rows .map( (r) => ` ${r.level} ${r.rule} ${r.message} *${escapeHtml(r.log.trim()) .split('*') .filter((el) => el) .join('
*')} ` ) .join('') return ` ${header} ${body}
` } /** * Return a body email */ buildEmailContent(result: TestResult): string { const rows = mapErrorsToRows(result.errors) const table = this.renderErrorTable(rows) return `

Cisco Device Log Result

Line: ${this.config.lineNumber} - Station: ${this.config.stationName}

Summary: ${result.summary}


${table}
` } /** * Update note of SN to ERP after run DPELP */ async updateNote(sn: string, data: DataDPELP) { const licenses = Array.isArray(data.license) ? [...new Set(data.license)] : data.license ? [data.license] : [] const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const dataFormat = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm') const note = `-------[ATC]-[${dataFormat}]-------\nLicense: ${licenses.join(', ')}\nSummary: ${data?.summary || ''}\nIssues:\n${data.issues?.length ? `- ` + data.issues.join(`\n- `) : ''}\n\n` await updateNoteToERP(sn, note) } /** * Update note of SN to ERP from user input */ async updateNoteFromUser(sn: string, note: string, licenses: string[]) { const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const dataFormat = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm') const data = `-------[ATC]-[${dataFormat}]-------\nLicense: ${licenses.join(', ')}\nIssues:\n${note}\n\n` const issueList = note .split('\n') .map((line) => (line[0] === '-' ? line.substring(1).trim() : line.trim())) const detectAI = this.config?.latestScenario?.detectAI ? { ...this.config.latestScenario.detectAI, issue: issueList } : { issue: issueList, summary: '', status: [] } if (this.config.latestScenario) { this.config.latestScenario = { ...this.config.latestScenario, detectAI } } await updateNoteToERP(sn, data) } /** * Starting physical test (PoE ports testing) */ async runPhysicalTest() { if (this.config.runningPhysical) { console.log('Running physical test') return } this.config.runningPhysical = true this.config.runningScenario = 'Physical Test' const listPorts = await this.getPorts() this.testingPortPoE = true this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: 'Physical Test', physical: true, ports: listPorts, }) if (listPorts.length === 0) { console.log('End physical test') this.endTesting() return } this.physicalTest.start(listPorts, this.config.inventory) // const interval = setInterval(async () => { // if (!this.physicalTest.done) { // // const result = this.physicalTest.getResult() // // console.warn('⚠️ Missing ports:', result.missingPorts) // } else { // clearInterval(interval) // await this.sendReportPhysicalTest() // this.endTesting() // } // }, 10000) } flushLogBuffer() { const lines = this.outputPhysicalTest.split(/\r?\n/) // giữ lại dòng cuối nếu chưa kết thúc hoàn chỉnh this.outputPhysicalTest = lines.pop() || '' const completeLines = lines.join('\n') if (completeLines.trim()) { const ports = this.physicalTest.handleLog(completeLines) if (ports?.length) this.socketIO.emit('test_port_physical', { stationId: this.config.stationId, lineId: this.config.id, data: ports, }) } } /** * End all testing */ endTesting() { this.physicalTest.done = true this.physicalTest.resetTestedPorts() this.config.runningPhysical = false this.config.runningScenario = '' this.testingPortPoE = false this.outputBuffer = '' this.outputScenario = '' this.outputPhysicalTest = '' this.config.ports = [] this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: '', }) } /** * Get list PoE ports */ async getPorts(): Promise { this.writeCommand(' show power inline\r\n') this.writeCommand(' \r\n') await this.sleep(5000) const statusOutput = this.outputPhysicalTest this.outputPhysicalTest = '' const lines = statusOutput.split('\n') const ports = [] for (const line of lines) { // Match: "Gi0/1 is up, line protocol is up" const match = line.match(/^(\S+)\s+\S+\s+(on|off)/i) if (match) { const name = match[1] ports.push(normalizeInterface(name)) } } this.config.ports = [...new Set(ports)] return [...new Set(ports)] } /** * Send report after done physical test */ async sendReportPhysicalTest() { const formReport = this.physicalTest.getFormReport() await sendMessageToMail( `[ATC] - [${this.config.stationName} - Line: ${this.config.lineNumber}] - Physical Port Test`, formReport ) } /** * Handle load ios for router */ async loadIosRouter(nameIos: string, userName: string) { const station = await Station.find(this.config.stationId) if (!station) return this.outputLoadIosLicense = true const network = station?.gateway || '172.25.1.1' const tftpIp = station?.tftp_ip || '172.16.7.69' const [a, b] = network.split('.').map(Number) const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const startTime = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm:ss') const body = buildBody( 'ROUTER_IOS', tftpIp, nameIos, `${a}.${b}.100.${this.config.id < 254 ? this.config.id : 254 - this.config.id}`, `${station?.gateway ? station?.gateway : '0.0.0.0'}`, this.listDeviceIos ) const script = { id: 0, isReboot: true, sendResult: false, send_result: false, title: 'Load IOS Router', timeout: 1000, body: JSON.stringify(body), } await sleep(5000) await this.runScript(script as any, userName) await this.sendEmailLoadIos(nameIos, startTime) } /** * Handle load ios for switch */ async loadIosSwitch(nameIos: string, userName: string) { const station = await Station.find(this.config.stationId) if (!station) return this.outputLoadIosLicense = true const network = station?.gateway || '172.25.1.1' const tftpIp = station?.tftp_ip || '172.16.7.69' const [a, b] = network.split('.').map(Number) const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const startTime = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm:ss') await this.backupIos(nameIos) const body = buildBody( 'SWITCH_IOS', tftpIp, nameIos, `${a}.${b}.100.${this.config.id < 254 ? this.config.id : 254 - this.config.id}`, `${station?.gateway ? station?.gateway : '0.0.0.0'}`, this.listDeviceIos ) const script = { id: 0, isReboot: false, sendResult: false, send_result: false, title: 'Load IOS Switch', timeout: 1000, body: JSON.stringify(body), } await this.runScript(script as any, userName) await this.sendEmailLoadIos(nameIos, startTime) } /** * Send mail report after load ios */ async sendEmailLoadIos(nameIos: string, startTime: string) { const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const dataFormat = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm:ss') const body = ` Load IOS Report
────────────────────────────────
Station : ${this.config.stationName}
Line : ${this.config.lineNumber}
IOS : ${nameIos}
Started At : ${startTime}
Finished At : ${dataFormat}

`.trim() await sendMessageToMail( `[ATC] - [${this.config.stationName} - Line: ${this.config.lineNumber}] - Load IOS Report`, body + `${`

Logs:

${this.outputLoadIosLicense}
`}` ) this.outputLoadIosLicense = '' } /** * Send mail report after load license */ async sendEmailLoadLicense(nameLicense: string, startTime: string) { const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const dataFormat = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm:ss') const report = parseLicenseReport( typeof this.outputLoadIosLicense === 'string' ? this.outputLoadIosLicense : '' ) const body = ` Load License Report
────────────────────────────────
Station : ${this.config.stationName}
Line : ${this.config.lineNumber}
License : ${nameLicense}
Started At : ${startTime}
Finished At : ${dataFormat}
────────────────────────────────
Summary licenses: ${report.summary.join(', ')}
Successful: ${report.imported.join(', ')}
Exist: ${report.exist.join(', ')}
Failed: ${report.failed.join(', ')}

`.trim() await sendMessageToMail( `[ATC] - [${this.config.stationName} - Line: ${this.config.lineNumber}] - Load License Report`, body + `${`

Logs:

${this.outputLoadIosLicense}
`}` ) this.outputLoadIosLicense = '' } /** * Check list ios exist on flash */ async checkDeviceFlash() { this.writeCommand(' enable\r\n') this.writeCommand('show flash:\r\n') await sleep(2000) const output = this.outputBuffer const ios: string[] = [] let match const SWITCH_BIN_REGEX = /^\s*\d+\s+-rwx\s+\d+\s+.*?\s+([^\s]+\.bin)\s*$/gim const ROUTER_BIN_REGEX = /^\s*\d+\s+(\d+)\s+.*?\s+([^\s]+\.bin)\s*$/gim // 🔍 Detect device type const isSwitch = output.includes('-rwx') const regex = isSwitch ? SWITCH_BIN_REGEX : ROUTER_BIN_REGEX // reset regex state regex.lastIndex = 0 while ((match = regex.exec(output)) !== null) { ios.push(isSwitch ? match[1] : match[2]) } return ios } /** * Delete File on Flash */ async deleteFileOnFlash(fileName: string) { await this.writeCommand(`delete flash:${fileName}\r\n`) await this.writeCommand(`\r\n`) await this.writeCommand(`\r\n`) await sleep(3000) } /** * Upload file from flash to TFTP server */ async uploadFileToServerTFTP(fileName: string, server: string) { this.config.runningScenario = 'Upload file' await this.writeCommand(`copy flash: tftp:\r\n`) await this.writeCommand(`${fileName}\r\n`) await this.writeCommand(`${server}\r\n`) await this.writeCommand(`i/${fileName}\r\n`) this.outputBuffer = '' await sleep(5000) while (true) { if (this.outputBuffer.includes('#')) { this.outputBuffer = '' this.config.runningScenario = '' return true } await sleep(5000) } } /** * Get list ios from TFTP server */ async getListIos() { try { const controller = new IosLicenseController() const listIos = await controller.getIos() return listIos } catch (error) { console.log('Error get ios', error) return [] } } /** * Get current boot ios of device */ async getCurrentBootIos() { this.writeCommand('show version | include System image\r\n') await sleep(2000) const match = this.outputBuffer.match(/"flash:(.+?)"/i) this.outputBuffer = '' return match ? match[1] : null } /** * Backup ios to TFTP, after that delete it on flash for free space */ async backupIos(nameIos: string) { const station = await Station.find(this.config.stationId) if (!station) return const server = station?.tftp_ip || '172.16.7.69' // const currentBootIos = await this.getCurrentBootIos() this.config.runningScenario = 'Backup IOS' this.socketIO.emit('running_scenario', { stationId: this.config.stationId, lineId: this.config.id, title: 'Backup IOS', }) await sleep(1000) const listIos = await this.getListIos() const dataDevice = await this.checkDeviceFlash() this.listDeviceIos = [...dataDevice] console.log('Data Device Flash', dataDevice) if (dataDevice && Array.isArray(dataDevice)) { for (const ios of dataDevice) { // if (ios === nameIos) { // console.log(`SKIP active IOS: ${ios}`) // continue // } if (listIos?.map((value) => value.name)?.includes(ios)) { console.log(`Already backed up: ${ios}`) if (ios !== nameIos) await this.deleteFileOnFlash(ios) } else { const ok = await this.uploadFileToServerTFTP(ios, server) if (ok && ios !== nameIos) await this.deleteFileOnFlash(ios) } } } this.outputBuffer = '' this.config.runningScenario = '' await sleep(1000) } /** * Handle load License for switch * Assumes traditional licensing (PAK/file-based) via TFTP */ async loadLicenseSwitch(licenseFileName: string, userName: string, portName: string) { const station = await Station.find(this.config.stationId) if (!station) return this.outputLoadIosLicense = true // Setup network variables (giống hệt logic load IOS để đảm bảo thông mạng) const network = station?.gateway || '172.25.1.1' const tftpIp = station?.tftp_ip || '172.16.7.69' const [a, b] = network.split('.').map(Number) // Setup time/logging const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const startTime = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm:ss') const body = buildBody( 'SWITCH_LICENSE', tftpIp, licenseFileName, `${a}.${b}.100.${this.config.id < 254 ? this.config.id : 254 - this.config.id}`, `${station?.gateway ? station?.gateway : '0.0.0.0'}`, this.listDeviceIos, portName ) const script = { id: 0, isReboot: false, sendResult: false, send_result: false, title: 'Load License Switch', timeout: 1000, body: JSON.stringify(body), } await this.runScript(script as any, userName) await this.sendEmailLoadLicense(licenseFileName, startTime) // Nếu bạn có hàm gửi mail báo cáo } /** * Handle load License for Router */ async loadLicenseRouter(licenseFileName: string, userName: string, portName: string) { const station = await Station.find(this.config.stationId) if (!station) return this.outputLoadIosLicense = true const network = station?.gateway || '172.25.1.1' const tftpIp = station?.tftp_ip || '172.16.7.69' const [a, b] = network.split('.').map(Number) const timeZone = process.env.TIME_ZONE || 'Australia/Sydney' const startTime = momentTZ().tz(timeZone).format('YYYY/MM/DD, HH:mm:ss') const body = buildBody( 'ROUTER_LICENSE', tftpIp, licenseFileName, `${a}.${b}.100.${this.config.id < 254 ? this.config.id : 254 - this.config.id}`, `${station?.gateway ? station?.gateway : '0.0.0.0'}`, this.listDeviceIos, portName ) const script = { id: 0, isReboot: false, sendResult: false, send_result: false, title: 'Load License Router', timeout: 1000, body: JSON.stringify(body), } await this.runScript(script as any, userName) await this.sendEmailLoadLicense(licenseFileName, startTime) } /** * Detect log by call api gpt, return string[] */ async detectShowEnvWithAI(log: string) { try { const payload = { model: 'gpt-4o-mini', max_tokens: 1000, messages: [ { role: 'user', content: `You are a network log parser. Input is the raw output of Cisco "show environment" or "show environment all". Your task: - Focus ONLY on FAN and POWER related information. - Ignore TEMPERATURE, VOLTAGE, and other sensors unless they relate to FAN or POWER. - Extract each FAN or POWER component and its state. - Normalize each item into the format: ": " Examples: - "FAN is OK" -> "FAN: OK" - "FAN 2 is FAILED" -> "FAN 2: FAILED" - "POWER SUPPLY A is NOT PRESENT" -> "POWER SUPPLY A: NOT PRESENT" - "PSU 1 Absent" -> "PSU 1: ABSENT" Output requirements: - Return ONLY a valid JSON array of strings. - Do NOT include any explanation or extra text. - Do NOT include code block. - JSON must be directly parsable. Here is the input log: ${log} `, }, ], } const remoteUrl = process.env.ERP_URL || 'https://stage.nswteam.net' const remoteResp = await axios.post( remoteUrl + '/api/transferPostData', { urlAPI: '/api/open-ai-sfp/model-image-info', data: payload, }, { headers: { Authorization: 'Bearer ' + process.env.ERP_TOKEN, }, } ) return remoteResp.data?.Status === 'OK' ? remoteResp.data?.data : '' } catch (error: any) { console.log('[ERROR] Detect log show env from AI', error) } return '' } async checkConfigRam(mem: string, flash: string, pid: string, output: string) { const configRam = await detectConfigRamByModel(pid) if (configRam) { const isWarningRAM = isRamSufficient(mem, configRam.ram) const isWarningFlash = isRamSufficient(flash, configRam.flash) if (isWarningRAM || isWarningFlash) { const subject = `[ATC] - [${this.config.stationName} - Line: ${this.config.lineNumber}] - Warning RAM, Flash Configuration` const body = `

Station: ${this.config.stationName}

Line: ${this.config.lineNumber}

Model: ${pid}

RAM: ${mem ? `${mem + ' bytes'} (default: ${configRam.ram})` : ''}

FLASH: ${flash ? `${flash + ' bytes'} (default: ${configRam.flash})` : ''}


${escapeHtml(output).replace('show ver', '').replace('sh ver', '').replace('show version', '').replace('sh version', '').replace(mem, `${mem}`).replace(flash, `${flash}`)}
` await sendMessageToMail(subject, body) } } } }