"""Hand-gesture tracker — standalone process, UDP JSON events for the HUD. Watches the webcam with MediaPipe Hands, tracks BOTH hands independently, classifies poses (point / pinch / fist / palm), and streams one JSON object per UDP datagram to 127.0.0.1:47831. Each hand is tagged by the USER's real hand — "R" or "L" (MediaPipe reports handedness from the mirrored image's POV, so the user's actual hand is the OPPOSITE; we normalize + debounce it): {"t":"hands","armed":true,"hands":[ ~30Hz stream {"hand":"R","x":0.42,"y":0.31,"pose":"point|pinch|mpinch|fist|palm|none"}, ...0..2 entries, one per hand SEEN this frame... ]} {"t":"gesture","name":"ready"} once at tracker start {"t":"gesture","name":"arm"|"disarm"} {"t":"gesture","name":"pinch_start","hand":"R","x":..,"y":..} STABILIZED coords {"t":"gesture","name":"pinch_move|pinch_end","hand":"R","x":..,"y":..} live coords {"t":"gesture","name":"rclick","hand":"R","x":..,"y":..} thumb+middle, one-shot {"t":"gesture","name":"fist_hold","hand":"R","x":..,"y":..} once per >=0.6s hold Each hand runs its OWN pinch state machine, its OWN thumb+middle right-click one-shot, and its OWN fist one-shot, so both hands can gesture at once and each emits its own stream tagged R/L (the Swift side decides policy: R = left-click). x/y are screen coords 0..1, origin TOP-LEFT, already mirrored (webcam is mirrored relative to the user), mapped through a PER-HAND AUTO-CENTERED window (SPAN_X/SPAN_Y wide, re-centered on the hand when it appears or when gestures (dis)arm — see _window_lo) and One-Euro smoothed. The cursor anchor is ALWAYS the index fingertip (even while pinching), so clicks land exactly where you point. pinch_start/rclick report a STABILIZED position from ~0.12s before the pinch, so the pinch's fingertip perturbation doesn't drag the click off-target. Safety: starts DISARMED. Hold an open palm ~1s to arm; again to disarm (palm must be dropped between toggles, plus a 1.5s cooldown). While disarmed nothing is sent except the arm/disarm events. Auto-disarms after 120s without a hand in view. Standalone by design (stdlib + cv2 + mediapipe + numpy only) so it can run under .venv-hands if the main venv ever can't hold mediapipe (control.py prefers that interpreter when present). HANDS_DEBUG=1 shows the annotated camera feed. """ from __future__ import annotations import json import math import os import signal import socket import sys import time from collections import deque from pathlib import Path import numpy as np UDP_ADDR = ("127.0.0.1", 47831) CAM_W, CAM_H, CAM_FPS = 640, 480, 30 # Which camera watches the hands. With TWO cameras (e.g. a desk cam aimed at # the hand resting on the table + a webcam used as the mic), point this at # the desk cam: HANDS_CAM=1 (or whatever index cv2 gives it). CAM_INDEX = int(os.environ.get("HANDS_CAM", "0")) # mediapipe >=0.10.3x ships only the Tasks API (mp.solutions is gone); # the landmarker model is a separate download, auto-fetched on first run MODEL = Path(__file__).resolve().parent.parent / "models" / "hand_landmarker.task" MODEL_URL = ("https://storage.googleapis.com/mediapipe-models/hand_landmarker/" "hand_landmarker/float16/latest/hand_landmarker.task") # pose thresholds, relative to hand size (= wrist -> middle-MCP distance). # The index pinch (left-click) and the thumb+middle pinch (right-click) share # the SAME relative-distance hysteresis. Fist-vs-pinch is NOT solved by a tight # distance threshold (that hurt intentional pinches); instead the classifier # GATES pinch entry on `not fist_shape` (see _update), so curling into a fist — # where thumb and index pass close — can never open a pinch. That lets the # thresholds stay generous/natural. PINCH_ENTER = 0.35 # thumb-tip distance (x hand size) to ENTER a pinch PINCH_EXIT = 0.55 # ...and to leave it (hysteresis) # EXTEND_MARGIN: a fingertip must sit this factor farther from the wrist than # its own PIP to count as "extended". Higher = fewer false point/palm reads # when the hand is far and landmark noise compresses the tip-vs-pip gap. EXTEND_MARGIN = 1.15 # tip this factor farther from wrist than its PIP THUMB_OUT = 0.45 # thumb tip distance from index MCP = thumb extended PALM_SPREAD = 0.65 # index-tip to pinky-tip distance for "spread" DEBOUNCE = 3 # frames a pose must persist before it's reported PALM_HOLD_S = 1.0 # palm hold that toggles armed TOGGLE_COOLDOWN_S = 1.5 AUTO_DISARM_S = 120.0 # no hand this long -> disarm FIST_HOLD_S = 0.6 # AUTO-CENTERED cursor window (replaces the old fixed X/Y_LO/HI box). Each hand # maps a SPAN-wide slice of the mirrored camera frame, centered on wherever the # hand was when it appeared / when gestures were (dis)armed, onto the full # screen. A tighter SPAN = higher gain, so the far screen corners are reachable # WITHOUT a tiring full-arm reach (the hand also stays in the camera's good # detection zone). The window is SHIFTED (never shrunk) to stay inside # [WINDOW_MIN, WINDOW_MAX] of the frame, so gain stays constant near the edges. # Tunable via env for different camera placements: a DESK CAM close to a # resting hand sees big hand motion per frame → RAISE the span (less # sensitivity, e.g. HANDS_SPAN_X=0.6); a far laptop cam → lower it. SPAN_X = float(os.environ.get("HANDS_SPAN_X", "0.36")) # full-width sweep SPAN_Y = float(os.environ.get("HANDS_SPAN_Y", "0.32")) # full-height sweep WINDOW_MIN, WINDOW_MAX = 0.02, 0.98 REAPPEAR_GAP_S = 0.8 # absence at least this long -> recapture the hand's center CURSOR_HIST = 12 # smoothed-cursor ring buffer length (~0.4s at 30fps) STABILIZE_LAG_S = 0.12 # pinch_start/rclick report the cursor this long BEFORE # the pinch, so the pinch's fingertip perturbation # doesn't drag the click off-target # per-hand cursor One-Euro tuning (fingertip). Retuned DOWN for the higher gain # of the tighter SPAN window: a lower min_cutoff is steadier at rest so small # targets stay clickable; beta keeps fast moves lag-free. CURSOR_MIN_CUTOFF = 1.4 CURSOR_BETA = 0.010 # handedness stickiness: a hand keeps its "R"/"L" id across frames (matched by # position) unless a differing MediaPipe classification persists this many # frames, so a 1-2 frame label flip never swaps the two reticles' identities. STICKY_FRAMES = 3 # MATCH_DIST: max normalized centroid move between frames still counted as the # SAME hand — a fast move keeps its track instead of resetting filters (which # would jump the reticle). MATCH_DIST = 0.40 # max normalized centroid move to count as the same hand WRIST, THUMB_TIP, INDEX_MCP, INDEX_TIP, MIDDLE_MCP, MIDDLE_TIP = 0, 4, 5, 8, 9, 12 FINGERS = ((8, 6), (12, 10), (16, 14), (20, 18)) # (tip, pip) index..pinky PINKY_TIP = 20 def _window_lo(center, span): """Low edge of the SPAN-wide auto-centered window, SHIFTED (not shrunk) to keep the whole window inside [WINDOW_MIN, WINDOW_MAX] of the frame.""" lo = center - span / 2.0 if lo < WINDOW_MIN: lo = WINDOW_MIN if lo + span > WINDOW_MAX: lo = WINDOW_MAX - span return lo class OneEuro: """One-Euro filter (Casiez et al. 2012) — low jitter at rest, low lag in motion. min_cutoff/beta per the paper's interactive-cursor tuning.""" def __init__(self, min_cutoff=1.0, beta=0.007, d_cutoff=1.0): self.min_cutoff, self.beta, self.d_cutoff = min_cutoff, beta, d_cutoff self.t_prev = self.x_prev = self.dx_prev = None @staticmethod def _alpha(cutoff, dt): tau = 1.0 / (2.0 * math.pi * cutoff) return 1.0 / (1.0 + tau / dt) def __call__(self, x, t): x = float(x) if self.t_prev is None: self.t_prev, self.x_prev, self.dx_prev = t, x, 0.0 return x dt = max(t - self.t_prev, 1e-6) dx = (x - self.x_prev) / dt a_d = self._alpha(self.d_cutoff, dt) dx_hat = a_d * dx + (1.0 - a_d) * self.dx_prev cutoff = self.min_cutoff + self.beta * abs(dx_hat) a = self._alpha(cutoff, dt) x_hat = a * x + (1.0 - a) * self.x_prev self.t_prev, self.x_prev, self.dx_prev = t, x_hat, dx_hat return x_hat def reset(self): self.t_prev = self.x_prev = self.dx_prev = None class Hand: def __init__(self): self.seen = False self.pinching = False # raw index-pinch hysteresis state self.mpinching = False # raw thumb+middle (right-click) hysteresis self.raw_pose, self.raw_n = "none", 0 self.pose = "none" # debounced self.fx = OneEuro(CURSOR_MIN_CUTOFF, CURSOR_BETA) self.fy = OneEuro(CURSOR_MIN_CUTOFF, CURSOR_BETA) self.x = self.y = 0.5 # smoothed screen coords (index fingertip) self.pinch_active = False # emitted pinch_start, awaiting pinch_end self.mpinch_armed = True # right-click one-shot re-arm flag self.fist_since, self.fist_fired = None, False self.center = None # auto-center of this hand's window (mirrored cam) self.raw_tip = None # last raw mirrored fingertip (for recapture) self.last_seen_t = None # last time this hand was updated (absence detect) self.hist = deque(maxlen=CURSOR_HIST) # (t, x, y) of the smoothed cursor def reset(self): # NOTE: does NOT clear pinch_active — a hand that vanishes mid-pinch # must still emit its pinch_end in the next _events pass. x/y, center, # last_seen_t and hist are kept too, so pinch_end lands on the last # known fingertip and a brief drop keeps the same auto-center. self.pinching = False self.mpinching = False self.raw_pose, self.raw_n, self.pose = "none", 0, "none" self.fx.reset() self.fy.reset() self.fist_since, self.fist_fired = None, False def stabilized(self, now): """Smoothed cursor from ~STABILIZE_LAG_S ago (closest buffered sample), so a pinch/rclick reports where the hand was AIMING before the pinch perturbed the fingertip. Falls back to the live cursor.""" if not self.hist: return self.x, self.y target = now - STABILIZE_LAG_S _t, bx, by = min(self.hist, key=lambda s: abs(s[0] - target)) return bx, by class LabelSticker: """Turn MediaPipe's per-frame handedness into a STABLE user-hand id. MediaPipe classifies handedness from the (mirrored) image's POV, so the user's real hand is the OPPOSITE of its label ("Left"->user "R"). Raw labels flip for a frame or two, which would swap the two reticles' identities; we guard against that by tracking each hand by centroid position across frames and only committing a new id once a differing classification persists STICKY_FRAMES frames. Camera-free / testable.""" def __init__(self): self.tracks = [] # [{"id","c":(x,y),"cand","cand_n"}], newest per frame @staticmethod def _user(mp_label): return "R" if mp_label == "Left" else "L" def assign(self, raw): # raw = list of (mp_label, pts) -> list of (user_id "R"/"L", pts) dets = [{"raw": self._user(mp), "c": (float(pts[:, 0].mean()), float(pts[:, 1].mean())), "pts": pts, "track": None} for mp, pts in raw] # greedy nearest-centroid match to last frame's tracks (each once) used = set() for d in dets: best, bi = None, None for i, tr in enumerate(self.tracks): if i in used: continue dk = math.dist(d["c"], tr["c"]) if best is None or dk < best: best, bi = dk, i if bi is not None and best <= MATCH_DIST: used.add(bi) d["track"] = bi new_tracks = [] for d in dets: if d["track"] is not None: tr = self.tracks[d["track"]] if d["raw"] == tr["id"]: tr["cand"], tr["cand_n"] = None, 0 elif d["raw"] == tr["cand"]: tr["cand_n"] += 1 if tr["cand_n"] >= STICKY_FRAMES: tr["id"], tr["cand"], tr["cand_n"] = d["raw"], None, 0 else: tr["cand"], tr["cand_n"] = d["raw"], 1 tr["c"] = d["c"] else: # fresh hand — trust its current classification outright tr = {"id": d["raw"], "c": d["c"], "cand": None, "cand_n": 0} d["id"] = tr["id"] new_tracks.append(tr) # two hands must never share an id — force the second one apart if len(new_tracks) == 2 and new_tracks[0]["id"] == new_tracks[1]["id"]: flip = "L" if new_tracks[1]["id"] == "R" else "R" new_tracks[1]["id"] = flip new_tracks[1]["cand"], new_tracks[1]["cand_n"] = None, 0 dets[1]["id"] = flip self.tracks = new_tracks return [(d["id"], d["pts"]) for d in dets] class Tracker: """Pose classification + gesture state machine, camera-free for tests: feed step() a list of (label, pts) with label = the USER's real hand as "R"/"L" and pts = (21,2) normalized array; every outgoing packet goes through the injected send(dict). Both hands are tracked independently — each keeps its own filters, its own pinch state machine, and its own fist one-shot, all tagged by "R"/"L" (feed two hands -> two independent streams). """ def __init__(self, send): self.send = send self.hands = {} # "R"/"L" -> Hand self.armed = False self.palm_since = None self.palm_rearmed = True # palm must drop between toggles self.last_toggle = -1e9 self.last_hand = None # last time any hand was seen def step(self, detected, now): for h in self.hands.values(): h.seen = False for label, pts in detected: h = self.hands.setdefault(label, Hand()) # recapture this hand's window center when it (re)appears after a # gap of absence (or on its very first frame). gap = None if h.last_seen_t is None else now - h.last_seen_t appeared = h.center is None or (gap is not None and gap >= REAPPEAR_GAP_S) h.seen = True h.last_seen_t = now self._update(h, pts, now, appeared) if detected: self.last_hand = now for h in self.hands.values(): if not h.seen and h.raw_pose != "none": h.reset() self._events(now) self._stream() # ---------- per-hand pose ---------- def _update(self, h, pts, now, capture_center=False): size = max(float(np.linalg.norm(pts[MIDDLE_MCP] - pts[WRIST])), 1e-6) # finger extension + thumb-out FIRST, so fist_shape can gate pinches. w = pts[WRIST] ext = [np.linalg.norm(pts[t] - w) > np.linalg.norm(pts[p] - w) * EXTEND_MARGIN for t, p in FINGERS] thumb_out = (np.linalg.norm(pts[THUMB_TIP] - pts[INDEX_MCP]) > THUMB_OUT * size) fist_shape = not any(ext) and not thumb_out # index pinch (left-click). A pinch may NOT ENTER while fist_shape — a # curling fist passes thumb & index close but must never click. An # already-open pinch keeps its normal hysteresis exit (don't kill a drag). pdist = float(np.linalg.norm(pts[THUMB_TIP] - pts[INDEX_TIP])) if h.pinching: if pdist > PINCH_EXIT * size: h.pinching = False elif not fist_shape and pdist < PINCH_ENTER * size: h.pinching = True # thumb+middle pinch (right-click) — same hysteresis, gated to NOT while # index-pinching and NOT while fist_shape. mdist = float(np.linalg.norm(pts[THUMB_TIP] - pts[MIDDLE_TIP])) if h.mpinching: if mdist > PINCH_EXIT * size: h.mpinching = False elif not fist_shape and not h.pinching and mdist < PINCH_ENTER * size: h.mpinching = True spread = np.linalg.norm(pts[INDEX_TIP] - pts[PINKY_TIP]) \ > PALM_SPREAD * size # pose priority: fist first (regardless of thumb-index distance), then # the two pinches, then point / palm / none. if fist_shape: pose = "fist" elif h.pinching: pose = "pinch" elif h.mpinching: pose = "mpinch" elif ext[0] and not any(ext[1:]): pose = "point" elif all(ext) and spread: pose = "palm" else: pose = "none" if pose == h.raw_pose: h.raw_n += 1 else: h.raw_pose, h.raw_n = pose, 1 if h.raw_n >= DEBOUNCE: h.pose = pose # cursor anchor is ALWAYS the index fingertip — even while pinching — so # the reticle stays put when the user pinches to click. Mapped through # the per-hand AUTO-CENTERED window (see _window_lo) at SPAN gain. tip = (float(pts[INDEX_TIP][0]), float(pts[INDEX_TIP][1])) mx = 1.0 - tip[0] # mirror x (webcam is mirrored); y is not mirrored my = tip[1] h.raw_tip = (mx, my) if capture_center or h.center is None: h.center = (mx, my) sx = min(max((mx - _window_lo(h.center[0], SPAN_X)) / SPAN_X, 0.0), 1.0) sy = min(max((my - _window_lo(h.center[1], SPAN_Y)) / SPAN_Y, 0.0), 1.0) h.x, h.y = h.fx(sx, now), h.fy(sy, now) h.hist.append((now, h.x, h.y)) # ring buffer for click stabilization # ---------- gesture state machine ---------- def _events(self, now): palm = any(h.seen and h.pose == "palm" for h in self.hands.values()) if not palm: self.palm_since = None self.palm_rearmed = True elif self.palm_since is None: self.palm_since = now if (palm and self.palm_rearmed and self.palm_since is not None and now - self.palm_since >= PALM_HOLD_S and now - self.last_toggle >= TOGGLE_COOLDOWN_S): self._toggle(now) if (self.armed and self.last_hand is not None and now - self.last_hand > AUTO_DISARM_S): self._toggle(now) # auto-disarm: hand gone too long if not self.armed: # cancel any in-flight gesture silently while disarmed for h in self.hands.values(): h.pinch_active = False h.mpinch_armed = True h.fist_since, h.fist_fired = None, False return # Per-hand, INDEPENDENT state machines — both hands can gesture at once # and each emits its own pinch_* / rclick / fist stream tagged R/L. A # hand that vanished this frame still runs here (it stays in self.hands) # so a pinch dropped by the hand disappearing still gets its pinch_end. for label, h in self.hands.items(): pinching_now = h.seen and h.pose == "pinch" if h.pinch_active: if pinching_now: self._gest("pinch_move", hand=label, x=round(h.x, 4), y=round(h.y, 4)) else: self._gest("pinch_end", hand=label, x=round(h.x, 4), y=round(h.y, 4)) h.pinch_active = False elif pinching_now: h.pinch_active = True sx, sy = h.stabilized(now) # STABILIZED pre-pinch aim point self._gest("pinch_start", hand=label, x=round(sx, 4), y=round(sy, 4)) # thumb+middle right-click — one-shot on ENTER, re-arms on release. # (Classifier already gates mpinch off while pinching / fist_shape.) mpinch_now = h.seen and h.pose == "mpinch" if mpinch_now and h.mpinch_armed: h.mpinch_armed = False rx, ry = h.stabilized(now) # STABILIZED, like pinch_start self._gest("rclick", hand=label, x=round(rx, 4), y=round(ry, 4)) elif not mpinch_now: h.mpinch_armed = True # fist held -> one-shot until released (re-arms on release) if h.seen and h.pose == "fist": if h.fist_since is None: h.fist_since = now elif not h.fist_fired and now - h.fist_since >= FIST_HOLD_S: h.fist_fired = True self._gest("fist_hold", hand=label, x=round(h.x, 4), y=round(h.y, 4)) else: h.fist_since, h.fist_fired = None, False def _toggle(self, now): self.armed = not self.armed self.last_toggle = now self.palm_since = None self.palm_rearmed = False # re-center every hand in view on the toggle, so the cursor is anchored # to wherever the hand actually is the moment gestures (dis)arm. for h in self.hands.values(): if h.seen and h.raw_tip is not None: h.center = h.raw_tip # disarm cancels any in-flight gesture silently (see the not-armed # branch in _events, which clears every hand's pinch/fist state) self.send({"t": "gesture", "name": "arm" if self.armed else "disarm"}) print(f"hands: {'armed' if self.armed else 'disarmed'}", flush=True) def _gest(self, name, **kw): self.send({"t": "gesture", "name": name, **kw}) def _stream(self): if not self.armed: return # disarmed = silent (arm/disarm events only) # one object per hand SEEN this frame (0, 1, or 2); omit vanished hands out = [{"hand": label, "x": round(h.x, 4), "y": round(h.y, 4), "pose": h.pose} for label, h in self.hands.items() if h.seen] self.send({"t": "hands", "armed": True, "hands": out}) # ---------- process wiring ---------- # 21-landmark hand skeleton (for the debug overlay) CONNS = ((0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8), (5, 9), (9, 10), (10, 11), (11, 12), (9, 13), (13, 14), (14, 15), (15, 16), (13, 17), (17, 18), (18, 19), (19, 20), (0, 17)) def _draw(cv2, frame, detected, tracker): hgt, wid = frame.shape[:2] for _label, pts in detected: px = (pts * (wid, hgt)).astype(int) for a, b in CONNS: cv2.line(frame, tuple(px[a]), tuple(px[b]), (0, 200, 255), 1) for p in px: cv2.circle(frame, tuple(p), 3, (0, 255, 0), -1) frame = cv2.flip(frame, 1) # mirrored view feels natural hdr = "ARMED" if tracker.armed else "disarmed" col = (0, 255, 0) if tracker.armed else (0, 0, 255) cv2.putText(frame, hdr, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, col, 2) row = 60 # one line per hand seen this frame: R/L + pose + cursor for label, h in tracker.hands.items(): if not h.seen: continue cv2.putText(frame, f"{label}: {h.pose} {h.x:.2f},{h.y:.2f}", (10, row), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2) row += 28 cv2.imshow("hands", frame) def _ensure_model() -> bool: if MODEL.is_file(): return True print("hands: downloading hand_landmarker model (~8MB, one time)...", flush=True) try: import urllib.request MODEL.parent.mkdir(exist_ok=True) urllib.request.urlretrieve(MODEL_URL, MODEL) return True except Exception as e: MODEL.unlink(missing_ok=True) print(f"hands: ERROR — model download failed: {e}", flush=True) return False def main() -> int: import cv2 import mediapipe as mp from mediapipe.tasks import python as mp_tasks from mediapipe.tasks.python import vision if not _ensure_model(): return 1 debug = os.environ.get("HANDS_DEBUG") == "1" # Retry the open for a few seconds: on first enable the app is still # showing the camera permission prompt, so the very first attempt is # denied — keep trying so we pick up the grant the moment it lands. cap = None for attempt in range(20): # ~20s cap = cv2.VideoCapture(CAM_INDEX) if cap.isOpened(): break cap.release() if attempt == 0: print("hands: waiting for camera permission (grant Jarvis in " "System Settings → Privacy → Camera)...", flush=True) time.sleep(1.0) if cap is None or not cap.isOpened(): print(f"hands: ERROR — cannot open camera {CAM_INDEX} (in use, or no " "Camera permission; HANDS_CAM= to pick another)", flush=True) return 1 cap.set(cv2.CAP_PROP_FRAME_WIDTH, CAM_W) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, CAM_H) cap.set(cv2.CAP_PROP_FPS, CAM_FPS) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) def send(obj): try: sock.sendto(json.dumps(obj, separators=(",", ":")).encode(), UDP_ADDR) except OSError: pass running = [True] signal.signal(signal.SIGTERM, lambda *_a: running.__setitem__(0, False)) signal.signal(signal.SIGINT, lambda *_a: running.__setitem__(0, False)) # Tasks API ships one fixed hand model (no model_complexity knob); the # float16 "full" bundle runs ~30fps at 640x480 on CPU without trouble. landmarker = vision.HandLandmarker.create_from_options( vision.HandLandmarkerOptions( base_options=mp_tasks.BaseOptions(model_asset_path=str(MODEL)), running_mode=vision.RunningMode.VIDEO, num_hands=2, min_hand_detection_confidence=0.6, min_tracking_confidence=0.5)) tracker = Tracker(send) sticker = LabelSticker() print("hands: tracking started", flush=True) # camera is open and the landmarker is built — tell the HUD the tracker is # live so the user gets immediate feedback ("says on AND something happens"). send({"t": "gesture", "name": "ready"}) t0 = time.monotonic() last_ts = -1 try: while running[0]: ok, frame = cap.read() if not ok: time.sleep(0.05) # camera hiccup — retry, don't spin continue rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) ts = max(int((time.monotonic() - t0) * 1000), last_ts + 1) last_ts = ts # detect_for_video needs strictly increasing ts res = landmarker.detect_for_video( mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb), ts) raw = [] for i, lms in enumerate(res.hand_landmarks): mp_label = "Right" if i < len(res.handedness) and res.handedness[i]: mp_label = res.handedness[i][0].category_name raw.append((mp_label, np.array([(p.x, p.y) for p in lms]))) # normalize to the user's real hand ("R"/"L") + de-flicker ids detected = sticker.assign(raw) tracker.step(detected, time.monotonic()) if debug: _draw(cv2, frame, detected, tracker) if cv2.waitKey(1) & 0xFF == 27: break finally: cap.release() landmarker.close() if debug: cv2.destroyAllWindows() sock.close() print("hands: stopped", flush=True) return 0 if __name__ == "__main__": sys.exit(main())