218 lines
7.7 KiB
Python
218 lines
7.7 KiB
Python
import os
|
|
import sys
|
|
import cv2
|
|
import numpy as np
|
|
from PyQt6.QtCore import Qt, QUrl, QTimer
|
|
from PyQt6.QtWidgets import (
|
|
QApplication,
|
|
QMainWindow,
|
|
QVBoxLayout,
|
|
QWidget,
|
|
QPushButton,
|
|
QLabel,
|
|
QDialog,
|
|
)
|
|
from PyQt6.QtWebEngineWidgets import QWebEngineView
|
|
from PyQt6.QtGui import QPixmap, QImage
|
|
from services.action_service import ActionService # ✅ JS version (fake click/type)
|
|
|
|
|
|
class FBWindow(QMainWindow):
|
|
def __init__(self, template_dir="templates", delay=0.1):
|
|
super().__init__()
|
|
self.template_dir = os.path.abspath(template_dir)
|
|
self.delay = delay
|
|
|
|
if not os.path.exists(self.template_dir):
|
|
raise FileNotFoundError(f"Template dir not found: {self.template_dir}")
|
|
|
|
# --- UI ---
|
|
self.setWindowTitle("FB Auto Vision Login")
|
|
self.resize(1200, 800)
|
|
|
|
self.web = QWebEngineView()
|
|
self.web.setUrl(QUrl("https://facebook.com"))
|
|
|
|
self.btn_detect = QPushButton("Detect Inputs")
|
|
self.btn_detect.clicked.connect(self.detect_inputs_from_view)
|
|
|
|
self.status = QLabel("Status: Ready")
|
|
self.status.setAlignment(Qt.AlignmentFlag.AlignLeft)
|
|
|
|
layout = QVBoxLayout()
|
|
layout.addWidget(self.web)
|
|
layout.addWidget(self.btn_detect)
|
|
layout.addWidget(self.status)
|
|
|
|
container = QWidget()
|
|
container.setLayout(layout)
|
|
self.setCentralWidget(container)
|
|
|
|
# --- Action service ---
|
|
self.action = ActionService(webview=self.web, delay=self.delay)
|
|
|
|
# ----------------------------------------------------
|
|
def capture_webview(self):
|
|
"""Chụp nội dung webview → numpy array BGR"""
|
|
pixmap = self.web.grab()
|
|
if pixmap.isNull():
|
|
return None
|
|
|
|
qimg = pixmap.toImage().convertToFormat(QImage.Format.Format_RGBA8888)
|
|
width, height = qimg.width(), qimg.height()
|
|
ptr = qimg.bits()
|
|
ptr.setsize(height * width * 4)
|
|
arr = np.frombuffer(ptr, np.uint8).reshape((height, width, 4))
|
|
bgr = cv2.cvtColor(arr, cv2.COLOR_RGBA2BGR)
|
|
return bgr
|
|
|
|
# ----------------------------------------------------
|
|
def detect_inputs_from_view(self):
|
|
"""Chụp ảnh webview và dò template (lọc trùng lặp)"""
|
|
screen = self.capture_webview()
|
|
if screen is None:
|
|
self.status.setText("Status: Unable to capture webview")
|
|
return
|
|
|
|
annotated = screen.copy()
|
|
regions = []
|
|
print("[INFO] Bắt đầu nhận diện template...")
|
|
|
|
# --- Duyệt toàn bộ thư mục con ---
|
|
for root, _, files in os.walk(self.template_dir):
|
|
folder_name = os.path.basename(root).lower()
|
|
|
|
for file in files:
|
|
if not file.lower().endswith((".png", ".jpg", ".jpeg")):
|
|
continue
|
|
|
|
template_path = os.path.join(root, file)
|
|
template = cv2.imread(template_path)
|
|
if template is None:
|
|
continue
|
|
|
|
res = cv2.matchTemplate(screen, template, cv2.TM_CCOEFF_NORMED)
|
|
threshold = 0.75
|
|
loc = np.where(res >= threshold)
|
|
|
|
for pt in zip(*loc[::-1]):
|
|
top_left = (int(pt[0]), int(pt[1]))
|
|
bottom_right = (
|
|
int(pt[0] + template.shape[1]),
|
|
int(pt[1] + template.shape[0]),
|
|
)
|
|
score = float(res[pt[1], pt[0]])
|
|
regions.append((folder_name, file, top_left, bottom_right, score))
|
|
|
|
# --- Lọc bớt trùng bằng Non-Max Suppression ---
|
|
filtered = self.non_max_suppression(regions, overlap_thresh=0.3)
|
|
|
|
if not filtered:
|
|
self.status.setText("[WARN] Không phát hiện được gì")
|
|
print("[WARN] Không phát hiện được gì")
|
|
return
|
|
|
|
# --- Vẽ preview ---
|
|
for folder_name, _, top_left, bottom_right, _ in filtered:
|
|
cv2.rectangle(annotated, top_left, bottom_right, (0, 255, 0), 2)
|
|
cv2.putText(
|
|
annotated,
|
|
folder_name,
|
|
(top_left[0], top_left[1] - 5),
|
|
cv2.FONT_HERSHEY_SIMPLEX,
|
|
0.6,
|
|
(0, 255, 0),
|
|
2,
|
|
)
|
|
|
|
self.status.setText(f"[INFO] Phát hiện {len(filtered)} vùng hợp lệ")
|
|
print(f"[INFO] Phát hiện {len(filtered)} vùng hợp lệ")
|
|
|
|
self.show_preview(annotated)
|
|
QTimer.singleShot(800, lambda: self.autofill_by_detection(filtered))
|
|
|
|
# ----------------------------------------------------
|
|
def non_max_suppression(self, regions, overlap_thresh=0.3):
|
|
"""Giảm trùng vùng detect"""
|
|
if not regions:
|
|
return []
|
|
|
|
boxes = []
|
|
for folder_name, file, top_left, bottom_right, score in regions:
|
|
x1, y1 = top_left
|
|
x2, y2 = bottom_right
|
|
boxes.append([x1, y1, x2, y2, score, folder_name, file])
|
|
boxes = sorted(boxes, key=lambda x: x[4], reverse=True)
|
|
|
|
pick = []
|
|
while boxes:
|
|
current = boxes.pop(0)
|
|
pick.append(current)
|
|
boxes = [
|
|
b
|
|
for b in boxes
|
|
if b[5] != current[5] # khác folder_name (chỉ giữ 1 mỗi loại)
|
|
and self.iou(b, current) < overlap_thresh
|
|
]
|
|
return [
|
|
(b[5], b[6], (int(b[0]), int(b[1])), (int(b[2]), int(b[3])), b[4])
|
|
for b in pick
|
|
]
|
|
|
|
# ----------------------------------------------------
|
|
def iou(self, boxA, boxB):
|
|
"""Intersection-over-Union"""
|
|
xA = max(boxA[0], boxB[0])
|
|
yA = max(boxA[1], boxB[1])
|
|
xB = min(boxA[2], boxB[2])
|
|
yB = min(boxA[3], boxB[3])
|
|
interArea = max(0, xB - xA) * max(0, yB - yA)
|
|
boxAArea = (boxA[2] - boxA[0]) * (boxA[3] - boxA[1])
|
|
boxBArea = (boxB[2] - boxB[0]) * (boxB[3] - boxB[1])
|
|
iou = interArea / float(boxAArea + boxBArea - interArea + 1e-5)
|
|
return iou
|
|
|
|
# ----------------------------------------------------
|
|
def autofill_by_detection(self, regions):
|
|
"""Tự động click/gõ text theo vùng detect"""
|
|
for folder_name, filename, top_left, bottom_right, score in regions:
|
|
print(f"[ACTION] {folder_name}: {filename} ({score:.2f})")
|
|
label = folder_name.lower()
|
|
|
|
if "user" in label or "email" in label:
|
|
print("[DO] Điền email...")
|
|
self.action.write_in_region(
|
|
top_left, bottom_right, "myemail@example.com"
|
|
)
|
|
|
|
elif "pass" in label:
|
|
print("[DO] Điền mật khẩu...")
|
|
self.action.write_in_region(top_left, bottom_right, "mypassword123")
|
|
|
|
elif "button" in label or "login" in label:
|
|
print("[DO] Click nút đăng nhập... (tạm comment)")
|
|
# self.action.click_center_of_region(top_left, bottom_right)
|
|
|
|
# ----------------------------------------------------
|
|
def show_preview(self, bgr_img):
|
|
"""Hiển thị preview trong PyQt6"""
|
|
rgb = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB)
|
|
h, w = rgb.shape[:2]
|
|
qimg = QImage(rgb.data, w, h, rgb.strides[0], QImage.Format.Format_RGB888)
|
|
pix = QPixmap.fromImage(qimg)
|
|
|
|
dlg = QDialog(self)
|
|
dlg.setWindowTitle("Detection Preview")
|
|
v_layout = QVBoxLayout(dlg)
|
|
lbl = QLabel()
|
|
lbl.setPixmap(pix.scaled(800, 600, Qt.AspectRatioMode.KeepAspectRatio))
|
|
v_layout.addWidget(lbl)
|
|
dlg.exec()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app = QApplication(sys.argv)
|
|
win = FBWindow()
|
|
win.show()
|
|
sys.exit(app.exec())
|