"""Gaze / face-direction detector using MediaPipe Face Mesh + iris tracking. Runs the webcam in a daemon background thread and exposes a single method: is_looking() -> bool Returns True when Ahmed's face appears directed at the camera (or when no face is detected — defaulting to "assume he's talking to Jarvis"). Returns False only when a face IS detected but the gaze is clearly directed away from the camera. Design constraints - CPU-light: processes every Nth frame only (SAMPLE_EVERY = 5, ~6 fps at 30 fps input) and runs at half resolution by default. - Camera unavailable: silently no-ops; is_looking() always returns True. - Thread-safe: the flag is a plain bool protected by threading.Lock. - No GUI windows are opened (headless OpenCV). Gaze estimation approach (no iris model required) Face Mesh gives 478 landmarks including the iris centres (landmarks 468 and 473 for left / right eye respectively) and the corners of each eye. We project the iris centre relative to the eye-corner span to get a normalised horizontal offset in [-1, 1]. Values outside a threshold mean the person is looking left/right. We also check the vertical tilt of the nose bridge vs eye line to catch strong downward or upward gaze. The thresholds are deliberately loose: the intent is to ignore Ahmed only when he is clearly staring at a second screen or looking down at his phone — not to react to small natural eye movements during conversation. """ from __future__ import annotations import threading import time from typing import Optional import numpy as np # Horizontal iris offset in normalised eye-width units outside which we # consider the person to be looking away left/right. _HORIZONTAL_THRESH = 0.30 # Nose-tip vs midpoint-of-eyes vertical ratio outside which we consider # the person to be looking strongly up or down. _VERTICAL_THRESH = 0.55 # Process every Nth camera frame (reduces CPU). _SAMPLE_EVERY = 5 # Resolution to resize captured frames to before running Face Mesh. _PROC_WIDTH = 640 _PROC_HEIGHT = 360 class GazeDetector: """Continuously sample the webcam and track whether the user is facing it. Usage:: gd = GazeDetector() gd.start() ... if gd.is_looking(): process_speech(text) ... gd.stop() """ def __init__(self, camera_index: int = 0) -> None: self._camera_index = camera_index self._looking = True # default: assume looking self._lock = threading.Lock() self._running = False self._thread: Optional[threading.Thread] = None self._available = False # set True once camera opens OK # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ def start(self) -> None: """Start the background detection thread (non-blocking).""" if self._thread is not None and self._thread.is_alive(): return self._running = True self._thread = threading.Thread( target=self._run, name="gaze-detector", daemon=True ) self._thread.start() def stop(self) -> None: """Signal the background thread to stop.""" self._running = False def is_looking(self) -> bool: """Return True if the user appears to be facing the camera.""" with self._lock: return self._looking @property def available(self) -> bool: """True once the camera was successfully opened.""" return self._available # ------------------------------------------------------------------ # Background thread # ------------------------------------------------------------------ def _run(self) -> None: try: self._run_inner() except Exception as exc: print(f" [gaze-detector] fatal error: {exc!r}; " "gaze check disabled (defaulting to looking=True)") with self._lock: self._looking = True def _run_inner(self) -> None: try: import cv2 import mediapipe as mp except ImportError as exc: print(f" [gaze-detector] import error: {exc}; " "install mediapipe + opencv-python-headless") return mp_face_mesh = mp.solutions.face_mesh # type: ignore[attr-defined] cap = cv2.VideoCapture(self._camera_index) if not cap.isOpened(): print(" [gaze-detector] camera unavailable — " "defaulting to looking=True") return self._available = True print(" [gaze-detector] camera opened — iris tracking active") face_mesh = mp_face_mesh.FaceMesh( static_image_mode=False, max_num_faces=1, refine_landmarks=True, # enables iris landmarks (468-477) min_detection_confidence=0.5, min_tracking_confidence=0.5, ) frame_idx = 0 try: while self._running: ret, frame = cap.read() if not ret: # Camera hiccup — back off briefly, keep trying. time.sleep(0.1) continue frame_idx += 1 if frame_idx % _SAMPLE_EVERY != 0: continue # Downscale for speed. small = cv2.resize(frame, (_PROC_WIDTH, _PROC_HEIGHT)) rgb = cv2.cvtColor(small, cv2.COLOR_BGR2RGB) results = face_mesh.process(rgb) if not results.multi_face_landmarks: # No face detected — assume looking. with self._lock: self._looking = True continue lm = results.multi_face_landmarks[0].landmark looking = self._classify_gaze(lm) with self._lock: self._looking = looking finally: face_mesh.close() cap.release() print(" [gaze-detector] stopped") # ------------------------------------------------------------------ # Gaze classification # ------------------------------------------------------------------ @staticmethod def _classify_gaze(landmarks) -> bool: """Return True if the face landmarks indicate the user is looking at the camera (or at least not clearly looking away). Landmark indices (MediaPipe Face Mesh with refine_landmarks=True): 33 left eye outer corner 133 left eye inner corner 362 right eye inner corner 263 right eye outer corner 468 left iris centre 473 right iris centre 1 nose tip 152 chin 10 forehead centre """ def pt(idx: int): lm = landmarks[idx] return np.array([lm.x, lm.y]) # ---- horizontal iris offset (left eye) ---- left_outer = pt(33) left_inner = pt(133) left_iris = pt(468) left_span = np.linalg.norm(left_inner - left_outer) if left_span > 1e-6: left_offset = (left_iris[0] - left_outer[0]) / left_span - 0.5 else: left_offset = 0.0 # ---- horizontal iris offset (right eye) ---- right_inner = pt(362) right_outer = pt(263) right_iris = pt(473) right_span = np.linalg.norm(right_inner - right_outer) if right_span > 1e-6: right_offset = (right_iris[0] - right_outer[0]) / right_span - 0.5 else: right_offset = 0.0 h_offset = abs((left_offset + right_offset) / 2.0) # ---- vertical check via nose vs eye midpoint ---- eye_mid_y = (pt(33)[1] + pt(263)[1]) / 2.0 nose_tip_y = pt(1)[1] chin_y = pt(152)[1] forehead_y = pt(10)[1] face_height = chin_y - forehead_y if face_height > 1e-6: v_ratio = (nose_tip_y - eye_mid_y) / face_height else: v_ratio = 0.0 looking_horizontally = h_offset <= _HORIZONTAL_THRESH looking_vertically = v_ratio <= _VERTICAL_THRESH return looking_horizontally and looking_vertically