import os import sys import cv2 import numpy as np from PyQt6.QtCore import Qt, QUrl, QTimer from PyQt6.QtWidgets import ( QApplication, QMainWindow, QVBoxLayout, QWidget, QPushButton, QLabel, QDialog, ) from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtGui import QPixmap, QImage from services.action_service import ActionService # ✅ JS version (fake click/type) class FBWindow(QMainWindow): def __init__(self, template_dir="templates", delay=0.1): super().__init__() self.template_dir = os.path.abspath(template_dir) self.delay = delay if not os.path.exists(self.template_dir): raise FileNotFoundError(f"Template dir not found: {self.template_dir}") # --- UI --- self.setWindowTitle("FB Auto Vision Login") self.resize(1200, 800) self.web = QWebEngineView() self.web.setUrl(QUrl("https://facebook.com")) self.btn_detect = QPushButton("Detect Inputs") self.btn_detect.clicked.connect(self.detect_inputs_from_view) self.status = QLabel("Status: Ready") self.status.setAlignment(Qt.AlignmentFlag.AlignLeft) layout = QVBoxLayout() layout.addWidget(self.web) layout.addWidget(self.btn_detect) layout.addWidget(self.status) container = QWidget() container.setLayout(layout) self.setCentralWidget(container) # --- Action service --- self.action = ActionService(webview=self.web, delay=self.delay) # ---------------------------------------------------- def capture_webview(self): """Chụp nội dung webview → numpy array BGR""" pixmap = self.web.grab() if pixmap.isNull(): return None qimg = pixmap.toImage().convertToFormat(QImage.Format.Format_RGBA8888) width, height = qimg.width(), qimg.height() ptr = qimg.bits() ptr.setsize(height * width * 4) arr = np.frombuffer(ptr, np.uint8).reshape((height, width, 4)) bgr = cv2.cvtColor(arr, cv2.COLOR_RGBA2BGR) return bgr # ---------------------------------------------------- def detect_inputs_from_view(self): """Chụp ảnh webview và dò template (lọc trùng lặp)""" screen = self.capture_webview() if screen is None: self.status.setText("Status: Unable to capture webview") return annotated = screen.copy() regions = [] print("[INFO] Bắt đầu nhận diện template...") # --- Duyệt toàn bộ thư mục con --- for root, _, files in os.walk(self.template_dir): folder_name = os.path.basename(root).lower() for file in files: if not file.lower().endswith((".png", ".jpg", ".jpeg")): continue template_path = os.path.join(root, file) template = cv2.imread(template_path) if template is None: continue res = cv2.matchTemplate(screen, template, cv2.TM_CCOEFF_NORMED) threshold = 0.75 loc = np.where(res >= threshold) for pt in zip(*loc[::-1]): top_left = (int(pt[0]), int(pt[1])) bottom_right = ( int(pt[0] + template.shape[1]), int(pt[1] + template.shape[0]), ) score = float(res[pt[1], pt[0]]) regions.append((folder_name, file, top_left, bottom_right, score)) # --- Lọc bớt trùng bằng Non-Max Suppression --- filtered = self.non_max_suppression(regions, overlap_thresh=0.3) if not filtered: self.status.setText("[WARN] Không phát hiện được gì") print("[WARN] Không phát hiện được gì") return # --- Vẽ preview --- for folder_name, _, top_left, bottom_right, _ in filtered: cv2.rectangle(annotated, top_left, bottom_right, (0, 255, 0), 2) cv2.putText( annotated, folder_name, (top_left[0], top_left[1] - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2, ) self.status.setText(f"[INFO] Phát hiện {len(filtered)} vùng hợp lệ") print(f"[INFO] Phát hiện {len(filtered)} vùng hợp lệ") self.show_preview(annotated) QTimer.singleShot(800, lambda: self.autofill_by_detection(filtered)) # ---------------------------------------------------- def non_max_suppression(self, regions, overlap_thresh=0.3): """Giảm trùng vùng detect""" if not regions: return [] boxes = [] for folder_name, file, top_left, bottom_right, score in regions: x1, y1 = top_left x2, y2 = bottom_right boxes.append([x1, y1, x2, y2, score, folder_name, file]) boxes = sorted(boxes, key=lambda x: x[4], reverse=True) pick = [] while boxes: current = boxes.pop(0) pick.append(current) boxes = [ b for b in boxes if b[5] != current[5] # khác folder_name (chỉ giữ 1 mỗi loại) and self.iou(b, current) < overlap_thresh ] return [ (b[5], b[6], (int(b[0]), int(b[1])), (int(b[2]), int(b[3])), b[4]) for b in pick ] # ---------------------------------------------------- def iou(self, boxA, boxB): """Intersection-over-Union""" xA = max(boxA[0], boxB[0]) yA = max(boxA[1], boxB[1]) xB = min(boxA[2], boxB[2]) yB = min(boxA[3], boxB[3]) interArea = max(0, xB - xA) * max(0, yB - yA) boxAArea = (boxA[2] - boxA[0]) * (boxA[3] - boxA[1]) boxBArea = (boxB[2] - boxB[0]) * (boxB[3] - boxB[1]) iou = interArea / float(boxAArea + boxBArea - interArea + 1e-5) return iou # ---------------------------------------------------- def autofill_by_detection(self, regions): """Tự động click/gõ text theo vùng detect""" for folder_name, filename, top_left, bottom_right, score in regions: print(f"[ACTION] {folder_name}: {filename} ({score:.2f})") label = folder_name.lower() if "user" in label or "email" in label: print("[DO] Điền email...") self.action.write_in_region( top_left, bottom_right, "myemail@example.com" ) elif "pass" in label: print("[DO] Điền mật khẩu...") self.action.write_in_region(top_left, bottom_right, "mypassword123") elif "button" in label or "login" in label: print("[DO] Click nút đăng nhập... (tạm comment)") # self.action.click_center_of_region(top_left, bottom_right) # ---------------------------------------------------- def show_preview(self, bgr_img): """Hiển thị preview trong PyQt6""" rgb = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB) h, w = rgb.shape[:2] qimg = QImage(rgb.data, w, h, rgb.strides[0], QImage.Format.Format_RGB888) pix = QPixmap.fromImage(qimg) dlg = QDialog(self) dlg.setWindowTitle("Detection Preview") v_layout = QVBoxLayout(dlg) lbl = QLabel() lbl.setPixmap(pix.scaled(800, 600, Qt.AspectRatioMode.KeepAspectRatio)) v_layout.addWidget(lbl) dlg.exec() if __name__ == "__main__": app = QApplication(sys.argv) win = FBWindow() win.show() sys.exit(app.exec())