ATC_SIMPLE/BACKEND/app/services/line_connection.ts

825 lines
24 KiB
TypeScript

import fs from 'node:fs'
import { textfsmResults } from './../ultils/templates/index.js'
import net from 'node:net'
import {
appendLog,
applyRules,
classifyLog,
cleanData,
detectScenarioByModel,
isValidJson,
LogStreamBuffer,
mapToLineFormat,
sleep,
TestSession,
} from '../ultils/helper.js'
import Scenario from '#models/scenario'
import path from 'node:path'
import axios from 'axios'
import redis from '@adonisjs/redis/services/main'
import Line from '#models/line'
type Inventory = {
pid: string
vid: string
sn: string
licenseLevel: string
licenseType: string
nextLicenseLevel: string
}
interface LineConfig {
id: number
port: number
lineNumber: number
ip: string
stationId: number
stationName: string
stationIp: string
apcName?: string
outlet: number
output: string
status: string
baud: number
openCLI: boolean
userEmailOpenCLI: string
userOpenCLI: string
inventory: any
latestScenario?: {
name: string
time: number
detectAI?: {
status: string[]
issue: string[]
summary: string
}
}
data: {
command: string
output: string
textfsm: string
}[]
commands: string[]
// history: string
}
/** HISTORY
* PID
* SN
* VID
* Timestamp
* Scenario
*/
interface HistoryItem {
pid: string
vid: string
sn: string
scenario: string
id: number
number: number
stationId: number
timestamp?: number
}
interface User {
userEmail: string
userName: string
}
interface DataDPELP {
line: number
pid: any
vid: any
sn: any
ios: string
mac: string
license: any
issues: string[]
summary: string
}
export default class LineConnection {
public client: net.Socket
public config: LineConfig
public readonly socketIO: any
private outputBuffer: string
private isRunningScript: boolean
private connecting: boolean
private waitingScenario: boolean
private outputInventory: string
private outputScenario: string
private bufferLog: LogStreamBuffer
public dataDPELP: DataDPELP | string
private listScenarios: number[]
public handleClearLine: () => void
private session: TestSession
constructor(config: LineConfig, socketIO: any, handleClearLine: () => void) {
this.config = config
this.socketIO = socketIO
this.client = new net.Socket()
this.outputBuffer = ''
this.isRunningScript = false
this.connecting = false
this.waitingScenario = false
this.outputInventory = ''
this.outputScenario = ''
this.bufferLog = new LogStreamBuffer()
this.dataDPELP = {
line: this.config.lineNumber,
pid: '',
vid: '',
sn: '',
ios: '',
mac: '',
license: [],
issues: ['No data'],
summary: '',
}
this.listScenarios = []
this.session = new TestSession()
this.handleClearLine = handleClearLine
}
connect(timeoutMs = 5000) {
return new Promise<void>((resolve, reject) => {
const { ip, port, lineNumber, id, stationId } = this.config
let resolvedOrRejected = false
// Set timeout
this.client.setTimeout(timeoutMs)
console.log(`🔌 Connecting to line ${lineNumber} (${ip}:${port})...`)
this.client.connect(port, ip, () => {
if (resolvedOrRejected) return
resolvedOrRejected = true
console.log(`[${Date.now()}] ✅ Connected to line ${lineNumber} (${ip}:${port})`)
this.connecting = true
setTimeout(() => {
this.config.status = 'connected'
// this.retryConnect = 0
this.connecting = false
this.socketIO.emit('line_connected', {
stationId,
lineId: id,
lineNumber,
status: 'connected',
})
// this.checkLog()
resolve()
}, 1000)
})
this.client.on('data', (data) => {
let message = this.connecting ? cleanData(data.toString()) : data.toString()
// const lines = this.bufferLog.push(data)
// lines.forEach(this.handleLogLine)
let rawData = ''
if (this.isRunningScript) {
this.waitingScenario = true
this.outputBuffer += message
this.outputScenario += message
if (!this.config.inventory)
this.outputInventory = this.outputInventory.slice(-3000) + message
}
if (data.toString().includes('More') || data.toString().includes('MORE'))
this.writeCommand(' ')
// let output = cleanData(message)
// console.log(`📨 [${this.config.port}] ${message}`)
// Handle netOutput with backspace support
for (const char of message) {
if (char === '\x7F' || char === '\x08') {
this.config.output = this.config.output.slice(0, -1)
// message = message.slice(0, -1)
} else {
rawData += char
}
}
this.config.output += cleanData(rawData)
this.config.output = this.config.output.slice(-15000)
this.socketIO.emit('line_output', {
stationId,
lineId: id,
data: message,
commands: this.config.commands,
})
if (!this.config.inventory) {
setTimeout(() => {
this.getInventory()
}, 5000)
}
appendLog(
cleanData(message),
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
})
this.client.on('error', (err) => {
if (resolvedOrRejected) return
resolvedOrRejected = true
console.error(`❌ Error line ${lineNumber}:`, err.message)
this.config.output += '\r\n' + err.message + '\r\n'
this.socketIO.emit('line_error', {
stationId,
lineId: id,
error: '\r\n' + err.message + '\r\n',
})
resolve()
})
this.client.on('close', async () => {
console.log(`[${Date.now()}] 🔌 Line ${lineNumber} disconnected`)
this.config.status = 'disconnected'
// this.config.inventory = undefined
this.socketIO.emit('line_disconnected', {
stationId,
lineId: id,
lineNumber,
status: 'disconnected',
})
// if (this.retryConnect <= 5) {
// await this.sleep(5000)
// console.log(`Retry connect line [${this.config.lineNumber}] times`, this.retryConnect)
// this.retryConnect += 1
// await this.reconnect()
// } else {
// this.retryConnect = 0
// }
})
this.client.on('timeout', () => {
if (resolvedOrRejected) return
resolvedOrRejected = true
const message = '\r\nConnection timeout!!\r\n'
this.config.output += message
this.socketIO.emit('line_output', {
stationId,
lineId: id,
data: message,
})
appendLog(
cleanData(message),
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
console.log(`⏳ Connection timeout line ${lineNumber}`)
this.client.destroy()
resolve()
// reject(new Error('Connection timeout'))
})
})
}
private sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
async writeCommand(cmd: string | Buffer<ArrayBuffer>, userName = '') {
if (this.client.destroyed) {
console.log(`⚠️ Cannot send, line ${this.config.lineNumber} is closed`)
this.disconnect()
// this.disconnect()
// await sleep(2000)
// await this.connect()
return
}
// console.log(
// `Write command "${cmd.toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}" to line ${this.config.lineNumber} of ${this.config.stationName}`
// )
this.client.write(cmd)
}
async disconnect() {
try {
console.log('[DISCONNECT] Line', this.config.lineNumber)
// this.handleClearLine()
this.client.destroy()
this.config.status = 'disconnected'
this.socketIO.emit('line_disconnected', {
...this.config,
status: 'disconnected',
})
console.log(`🔻 Closed connection to line ${this.config.lineNumber}`)
} catch (e) {
console.error('Error closing line:', e)
}
}
async runScript(script: Scenario, userName: string) {
if (!this.client || this.client.destroyed) {
console.log('Not connected')
this.isRunningScript = false
this.socketIO.emit('running_scenario', {
stationId: this.config.stationId,
lineId: this.config.id,
title: '',
})
this.outputBuffer = ''
return
}
if (this.isRunningScript) {
console.log('Script already running')
return
}
console.log(
`Run scenario "${script?.title}" to line ${this.config.lineNumber} of ${this.config.stationName}`
)
this.isRunningScript = true
this.socketIO.emit('running_scenario', {
stationId: this.config.stationId,
lineId: this.config.id,
title: script?.title,
})
if (script?.send_result || script?.sendResult) {
this.dataDPELP = ''
this.config.inventory = ''
}
if (script?.isReboot) {
await sleep(10000)
for (let index = 0; index < 30; index++) {
await sleep(1000)
this.breakSpam()
}
}
const now = Date.now()
this.outputScenario += `\n\n---start-scenarios---${now}---${userName}---${script?.title}---\n---scenario---${script?.title}---${now}---\n`
appendLog(
`\n\n---start-scenarios---${now}---${userName}---${script?.title}---\n---scenario---${script?.title}---${now}---\n`,
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
this.config.latestScenario = {
name: script?.title,
time: now,
}
const steps = typeof script?.body === 'string' ? JSON.parse(script?.body) : []
let stepIndex = 0
return new Promise((resolve, reject) => {
const timeoutTimer = setTimeout(
() => {
this.isRunningScript = false
this.socketIO.emit('running_scenario', {
stationId: this.config.stationId,
lineId: this.config.id,
title: '',
})
this.outputBuffer = ''
this.outputScenario = ''
this.config.output += 'Timeout run scenario'
this.dataDPELP = {
line: this.config.lineNumber,
pid: '',
vid: '',
sn: '',
ios: '',
mac: '',
license: [],
issues: ['No data'],
summary: '',
}
this.socketIO.emit('line_output', {
stationId: this.config.stationId,
lineId: this.config.id,
data: 'Timeout run scenario',
})
this.outputScenario += `\n---end-scenarios---${now}---${userName}---\n`
appendLog(
`\n---end-scenarios---${now}---${userName}---\n`,
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
// reject(new Error('Script timeout'))
},
script.timeout ? Number(script.timeout) * 1000 : 300000
)
const runStep = async (index: number) => {
if (index >= steps.length) {
if (this.waitingScenario) {
this.waitingScenario = false
setTimeout(() => {
runStep(index)
}, 5000)
return
} else clearTimeout(timeoutTimer)
this.isRunningScript = false
this.socketIO.emit('running_scenario', {
stationId: this.config.stationId,
lineId: this.config.id,
title: '',
})
this.outputBuffer = ''
this.outputScenario += `\n---end-scenarios---${now}---${userName}---\n`
appendLog(
`\n---end-scenarios---${now}---${userName}---\n`,
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
const logScenarios = this.outputScenario
const data = textfsmResults(logScenarios, '')
let pid = this.config.inventory?.pid || ''
try {
data.forEach((item) => {
if (item?.textfsm && isValidJson(item?.textfsm)) {
if (
['show inventory', 'sh inventory', 'show inv', 'sh inv'].includes(item.command)
) {
const dataInventory = JSON.parse(item.textfsm)[0]
this.config.inventory = dataInventory
pid = dataInventory?.pid || ''
this.addHistory(this.config.stationId, this.config.id, {
id: this.config.id,
number: this.config.lineNumber,
stationId: this.config.stationId,
pid: dataInventory?.pid,
sn: dataInventory?.sn,
vid: dataInventory?.vid,
scenario: script?.title,
timestamp: Date.now(),
})
}
item.textfsm = JSON.parse(item.textfsm)
}
})
const scenario = await detectScenarioByModel(pid, this.listScenarios)
console.log(pid, scenario?.title, this.listScenarios)
if (
scenario &&
scenario.id !== script.id &&
scenario.title.includes('DPELP') &&
script.title.includes('DPELP')
) {
this.listScenarios.push(scenario.id)
this.outputScenario = ''
this.runScript(scenario, userName)
// this.socketIO.emit('confirm_scenario', {
// scenario: scenario,
// id: this.config.id,
// })
resolve(true)
return
}
const detectLog = await this.detectLogWithAI(logScenarios)
const result = mapToLineFormat({
lineNumber: this.config.lineNumber,
inventory: this.config.inventory,
latestScenario: {
detectAI: detectLog,
},
data,
})
// if (script?.send_result || script?.sendResult) {
this.dataDPELP = result
console.log(
`DPELP DATA line ${this.config.lineNumber} of ${this.config.stationName}:`,
this.dataDPELP
)
// }
if (this.config.latestScenario)
this.config.latestScenario = { ...this.config.latestScenario, detectAI: detectLog }
this.config.data = data
this.socketIO.emit('data_textfsm', {
stationId: this.config.stationId,
lineId: this.config.id,
data,
inventory: this.config.inventory || null,
latestScenario: this.config.latestScenario || null,
})
} catch (error) {
console.log(error)
}
this.listScenarios = []
this.outputScenario = ''
resolve(true)
return
}
const step = steps[index]
let repeatCount = Number(step.repeat) || 1
const delay = step?.delay ? Number(step?.delay) * 1000 : 1000
const sendCommand = async () => {
if (repeatCount <= 0) {
// Done → next step
stepIndex++
return runStep(stepIndex)
}
if (typeof step.send !== 'undefined') {
this.outputScenario += `\n---send-command---"${(step?.send ?? '[ENTER]').toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}"---${now}---\n`
appendLog(
`\n---send-command---"${(step?.send ?? '[ENTER]').toString().replace(/\r/g, '\\r').replace(/\n/g, '\\n')}"---${now}---\n`,
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
this.writeCommand((step?.send || '') + '\r\n')
}
repeatCount--
setTimeout(() => sendCommand(), delay)
}
// Nếu expect rỗng → gửi ngay
if (!step?.expect || step?.expect.trim() === '') {
setTimeout(() => sendCommand(), delay)
return
}
// while (this.outputBuffer) {
// await sleep(200)
// if (this.outputBuffer.includes(step.expect)) {
// this.outputBuffer = ''
// setTimeout(() => sendCommand(), delay)
// }
// }
const matched = await this.waitForExpect(
step.expect.trim(),
script?.timeout ? Number(script?.timeout) * 1000 : 60000
)
if (matched) setTimeout(() => sendCommand(), delay)
}
runStep(stepIndex)
})
}
public async reconnect(): Promise<boolean> {
try {
this.disconnect()
this.client = new net.Socket()
await this.sleep(1000)
await this.connect()
return true
} catch (err: any) {
return false
}
}
userOpenCLI(user: User) {
this.config.openCLI = true
this.config.userEmailOpenCLI = user.userEmail
this.config.userOpenCLI = user.userName
this.socketIO.emit('user_open_cli', {
stationId: this.config.stationId,
lineId: this.config.id,
userEmailOpenCLI: user.userEmail,
userOpenCLI: user.userName,
})
appendLog(
`\n-------${user.userName}-------\n`,
this.config.stationId,
this.config.stationName,
this.config.stationIp,
this.config.lineNumber
)
}
userCloseCLI() {
this.config.openCLI = false
this.config.userEmailOpenCLI = ''
this.config.userOpenCLI = ''
this.socketIO.emit('user_close_cli', {
stationId: this.config.stationId,
lineId: this.config.id,
userEmailOpenCLI: '',
})
}
waitForExpect = async (expect: string, timeout = 60000) => {
const start = Date.now()
// console.log('[EXPECT]', expect, timeout)
while (Date.now() - start < timeout) {
if (this.outputBuffer.includes(expect)) {
this.outputBuffer = ''
return true
}
await sleep(200)
}
return false
}
getInventory = () => {
const data = textfsmResults(this.outputInventory, 'show inventory')
try {
data.forEach((item) => {
if (item?.textfsm && isValidJson(item?.textfsm)) {
if (['show inventory', 'sh inventory', 'show inv', 'sh inv'].includes(item.command)) {
const dataInventory = JSON.parse(item.textfsm)[0]
this.config.inventory = dataInventory
}
item.textfsm = JSON.parse(item.textfsm)
}
})
if (this.config.inventory) {
this.config.data = data
this.socketIO.emit('data_textfsm', {
stationId: this.config.stationId,
lineId: this.config.id,
data,
inventory: this.config.inventory || null,
latestScenario: this.config.latestScenario || null,
})
this.outputInventory = ''
}
} catch (error) {
console.log(error)
}
}
// Gửi nhiều ký tự ESC để vào ROMMON
breakSpam() {
console.log('SPAM Break to line:', this.config.lineNumber)
let count = 0
const escInterval = setInterval(() => {
if (count >= 10) {
clearInterval(escInterval)
return
}
this.client.write(Buffer.from([0xff, 0xf3])) // Ctrl + Break
count++
}, 1)
}
async setBaud(baud: number) {
this.writeCommand('enable\r\n')
await sleep(500)
this.writeCommand('configure terminal\r\n')
await sleep(500)
this.writeCommand('line console 0\r\n')
await sleep(500)
this.writeCommand(`speed ${baud.toString()}\r\n`)
await sleep(500)
this.writeCommand('end\r\n')
await sleep(500)
this.writeCommand('write memory\r\n')
this.writeCommand('\r\n')
}
async getLog(date: string) {
const logDir = path.join('storage', 'system_logs')
const logFile = path
.join(
logDir,
`${date}-AUTO-Session.${this.config.stationName}-${this.config.stationId}-${this.config.stationIp}-${this.config.lineNumber}.log`
)
.replaceAll(' ', '_')
if (!fs.existsSync(logDir) || !fs.existsSync(logFile)) {
return ''
}
return await fs.promises.readFile(logFile, 'utf8')
}
async detectLogWithAI(log: string) {
try {
const payload = {
model: 'gpt-4o-mini',
max_tokens: 1000,
messages: [
{
role: 'user',
content: `You are a network hardware tester.
Your task is to analyze router/switch logs to determine whether the device meets hardware standards for reselling.
Focus ONLY on hardware-related problems or abnormal warnings.
Software or configuration issues (e.g., port up/down, admin down, invalid commands, CLI errors, licensing messages) should be ignored unless they indicate hardware failure.
OUTPUT FORMAT (must follow exactly):
{
"issue": [ "problem 1", "problem 2", ... ],
"summary": "short summary under 30 words"
}
RULES:
- Summaries must be in English.
- Each issue must be one short line.
- If the log contains no hardware issues, output: { "issue": ["No issues detected."], "summary": "No hardware issues found." }
- Keep responses concise, readable, and strictly in JSON format.
- Do NOT add explanations outside the JSON.
- Your job is to detect hardware faults, missing components, overheating, failing modules, PSU issues, sensor anomalies, SIM/card missing, modem errors, transceiver issues, POST/diagnostics failures, etc.
The log to analyze will be provided after this prompt.
Here is the log:
${log}
`,
},
],
}
const remoteUrl = process.env.ERP_URL || 'https://stage.nswteam.net'
const remoteResp = await axios.post(
remoteUrl + '/api/transferPostData',
{
urlAPI: '/api/open-ai-sfp/model-image-info',
data: payload,
},
{
headers: {
Authorization: 'Bearer ' + process.env.ERP_TOKEN,
},
}
)
return remoteResp.data?.Status === 'OK' ? remoteResp.data?.data : ''
} catch (error: any) {
console.log('[ERROR] Detect log from AI', error)
}
return ''
}
async addHistory(stationId: number, lineId: number, item: HistoryItem) {
if (!item.pid || !item.sn) return
const key = `station:${stationId}:line:${lineId}:history`
const now = Date.now()
const newItem = JSON.stringify({
...item,
timestamp: now,
})
// Lấy phần tử cuối
const lastItems = await redis.zrevrange(key, 0, 0)
if (lastItems.length > 0) {
const last = JSON.parse(lastItems[0])
if (last.pid === item.pid && last.sn === item.sn) {
return false // không thay đổi
}
}
const line = await Line.find(lineId)
if (line) {
const listHistory = line.history ? JSON.parse(line.history) : []
listHistory.unshift(newItem)
line.history = JSON.stringify(listHistory)
await line.save()
}
// Add vào ZSET
await redis.zadd(key, now, newItem)
// Tự động xóa item > 96h
const expireTime = now - 96 * 60 * 60 * 1000
await redis.zremrangebyscore(key, 0, expireTime)
return true
}
async getHistory(stationId: number, lineId: number) {
const key = `station:${stationId}:line:${lineId}:history`
const items = await redis.zrange(key, 0, -1)
return items.map((i) => JSON.parse(i))
}
handleLogLine = (line: string) => {
try {
const parsed = classifyLog(line)
this.session.applyParsedLog(parsed)
} catch (error) {
console.log('handleLogLine', error)
}
}
checkLog() {
const interval = setInterval(async () => {
try {
if (this.config.status !== 'connected') {
clearInterval(interval)
return
}
const result = this.session.finalize()
console.log('===== TEST RESULT =====')
console.log('STATUS:', result.status)
console.log('SUMMARY:', result.summary)
result.errors.forEach((err, idx) => {
console.log(`\n[${idx + 1}] ${err.level} - ${err.ruleId}`)
console.log('Message:', err.message)
console.log('Log:', err.evidence.raw)
})
} catch (err: any) {
console.error('Error checking log:', err)
}
}, 30000)
}
}