package com.ultron.jarvistv import android.accessibilityservice.AccessibilityService import android.accessibilityservice.GestureDescription import android.graphics.Path import android.os.Handler import android.os.Looper import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo /** * The "hands" of Jarvis. Once enabled in Settings > Accessibility, this service can read the * current screen and perform taps / scrolls / text entry inside ANY app — that's what gives us * full cross-app control without root. A static reference lets ControlService drive it. */ class JarvisAccessibilityService : AccessibilityService() { companion object { @Volatile var instance: JarvisAccessibilityService? = null } private val main = Handler(Looper.getMainLooper()) override fun onServiceConnected() { super.onServiceConnected() instance = this } override fun onDestroy() { instance = null super.onDestroy() } // We don't need to react to every event for command-driven control. override fun onAccessibilityEvent(event: AccessibilityEvent?) {} override fun onInterrupt() {} // ---- High-level actions ControlService calls ---- fun back() = performGlobalAction(GLOBAL_ACTION_BACK) fun home() = performGlobalAction(GLOBAL_ACTION_HOME) fun recents() = performGlobalAction(GLOBAL_ACTION_RECENTS) /** Click the first node whose visible text (or content-description) contains [text]. */ fun clickText(text: String): Boolean { val root = rootInActiveWindow ?: return false val node = findByText(root, text.lowercase()) ?: return false return clickNode(node) } /** Type into the currently focused editable field. */ fun typeText(text: String): Boolean { val root = rootInActiveWindow ?: return false val field = findFocusedEditable(root) ?: return false val args = android.os.Bundle() args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) return field.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) } /** Tap absolute screen coordinates (fallback when there's no addressable node). */ fun tap(x: Float, y: Float) { val path = Path().apply { moveTo(x, y) } val stroke = GestureDescription.StrokeDescription(path, 0, 60) dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null) } fun scrollDown() { val root = rootInActiveWindow ?: return findScrollable(root)?.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD) } // ---- node helpers ---- private fun clickNode(start: AccessibilityNodeInfo): Boolean { var node: AccessibilityNodeInfo? = start while (node != null) { if (node.isClickable) return node.performAction(AccessibilityNodeInfo.ACTION_CLICK) node = node.parent } // last resort: tap its center val r = android.graphics.Rect() start.getBoundsInScreen(r) tap(r.exactCenterX(), r.exactCenterY()) return true } private fun findByText(node: AccessibilityNodeInfo, needle: String): AccessibilityNodeInfo? { val t = node.text?.toString()?.lowercase() val d = node.contentDescription?.toString()?.lowercase() if ((t != null && t.contains(needle)) || (d != null && d.contains(needle))) return node for (i in 0 until node.childCount) { val child = node.getChild(i) ?: continue findByText(child, needle)?.let { return it } } return null } private fun findFocusedEditable(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { if (node.isEditable && node.isFocused) return node for (i in 0 until node.childCount) { val child = node.getChild(i) ?: continue findFocusedEditable(child)?.let { return it } } // fall back to the first editable field on screen if (node.isEditable) return node return null } private fun findScrollable(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { if (node.isScrollable) return node for (i in 0 until node.childCount) { val child = node.getChild(i) ?: continue findScrollable(child)?.let { return it } } return null } /** Run a block on the main thread after [delayMs] (UI settling between steps). */ fun delayed(delayMs: Long, block: () -> Unit) = main.postDelayed(block, delayMs) }