"""Claude Voice — real-time, interruptible voice chat with Claude on a Mac. Everything runs locally on the M4 except the Claude conversation itself, which goes through the locally-installed Claude Code (Max subscription, no API key, no per-token billing). mic 16k ── Silero VAD ── Parakeet STT ─▶ Claude (Agent SDK, streaming) │ sentence chunks speakers 24k ◀── Speaker.cut() on barge-in ── Kokoro TTS State machine (parameters from what shipped voice projects converge on): LISTENING VAD 0.60 start / 250ms min speech / 700ms end-silence, 1s pre-roll ring so the first syllable isn't lost. RESPONDING streaming LLM -> per-sentence TTS -> playback. Barge-in: while audio is playing the VAD runs STRICT (0.90, ~290ms sustained, 300ms refractory) so the assistant doesn't trigger on its own voice from open speakers; before audio starts it runs at the normal threshold. On barge-in: playback cut (<50ms), TTS queue dropped, Claude generation interrupted, back to LISTENING. Env knobs: MODEL=haiku|sonnet|opus VOICE=af_heart VOICE_SPEED=1.0 STT=parakeet|whisper STT_LANG=ar INTERRUPT=0 (half-duplex) """ from __future__ import annotations import asyncio import collections import os import random import re import sys import threading import time # Windows: a piped stdout (e.g. spawned by the HUD) defaults to cp1252, and # any Unicode in a print (≥, arrows, box glyphs) kills the whole engine with # UnicodeEncodeError. Force UTF-8 with replacement so a print can never crash. if os.name == "nt": for _stream in (sys.stdout, sys.stderr): try: _stream.reconfigure(encoding="utf-8", errors="replace") except Exception: # noqa: BLE001 pass # Per-monitor DPI awareness: without this, Windows lies to us about # coordinates on scaled displays (mouse clicks land off-target on an # ultrawide at 125%). With it, screenshots, GetSystemMetrics, pyautogui # and SendInput all speak the same PHYSICAL pixels. try: import ctypes ctypes.windll.shcore.SetProcessDpiAwareness(2) # PER_MONITOR_AWARE except Exception: # noqa: BLE001 try: ctypes.windll.user32.SetProcessDPIAware() except Exception: # noqa: BLE001 pass import numpy as np # Load .env (mailbox creds, Google OAuth paths, Supabase, etc.) into the # process before any voice module reads os.environ. Best-effort: no .env or no # python-dotenv just means those integrations stay disabled. try: from dotenv import load_dotenv load_dotenv(os.path.join(os.path.dirname(__file__), ".env")) except Exception: # noqa: BLE001 pass from voice.audio_io import ( FRAME_SAMPLES, SAMPLE_RATE, AECMicStream, MicStream, Speaker, SystemAudioStream, _start_mute_hotkey, break_requested, restore_input_volume, ) from voice import entity_show from voice import pending, task_ledger from voice.control import ControlWatcher from voice.events import emit from voice.llm import ClaudeBrain, FastBrain from voice.frustration import FrustrationMeter from voice.prosody import ProsodyReader from voice.speaker_id import SpeakerGate from voice.stt import load_stt from voice.tts import SAMPLE_RATE as TTS_RATE, KokoroTTS from voice.vad import VAD FRAME_MS = FRAME_SAMPLES * 1000 // SAMPLE_RATE # 32 ms # LISTENING (end-of-utterance) # START_PROB 0.50 (was 0.60): headset hardware noise-gates clip the first # syllable of quiet speech — trigger earlier so gated onsets aren't lost; # MIN_SPEECH_MS still filters noise bursts. VAD_START env to tune. START_PROB = float(os.environ.get("VAD_START", "0.50")) END_PROB = 0.35 MIN_SPEECH_MS = 250 END_SILENCE_MS = int(os.environ.get("END_SILENCE_MS", "480")) # snappier turn-taking (was 700) # Semantic turn-taking (Smart Turn v3): after a SHORT pause, ask the model # whether the sentence sounded finished — end instantly if yes, keep # listening through thinking-pauses if no (up to the hard cap below). TURN_FIRST_CHECK_MS = 220 # first semantic check into a pause TURN_RECHECK_MS = 240 # re-check cadence while he might continue TURN_MAX_SILENCE_MS = int(os.environ.get("TURN_MAX_SILENCE_MS", "1400")) MAX_UTTER_S = 15 # force-end a runaway utterance (noise/clipping guard) _FEED_FRAMES = 13 # ~0.42s: how often to push audio to the live STT stream PRE_ROLL_FRAMES = 31 # ~1 s # RESPONDING (barge-in) — tuned per mic path in VoiceApp._pick_mic(): # AEC mic: the assistant's own voice is already cancelled, so anything # the VAD hears while playing is really the user -> relaxed settings. # Raw mic: speaker audio leaks straight into the mic (proven: VAD 1.00 # at full volume) -> very strict, and still unreliable on speakers. BARGE_PROB_QUIET = 0.60 # nothing playing (thinking gap) # Ignore triggers right after playback starts (guards a click/onset). Short # by default so interrupting the FIRST word of a sentence still works. REFRACTORY_S = float(os.environ.get("BARGE_REFRACTORY", "0.12")) INTERRUPTIBLE = os.environ.get("INTERRUPT", "1") != "0" # ASYNC BRAIN (chat-supervisor): a full-brain turn runs as a BACKGROUND task # instead of holding the floor — the mic comes back ~immediately, the fast # lane keeps the conversation (it's told a task is in flight), a second # action request queues, and the finished reply is announced through the # same idle-floor rail worker reports already use. ASYNC_BRAIN=0 restores # the old blocking turn exactly. ASYNC_BRAIN = os.environ.get("ASYNC_BRAIN", "1") != "0" # TASK-AWARE FILLER: how long the fast local model gets to produce a # task-specific ack ("Checking your emails now, sir.") before we speak a # generic canned line instead. Tight so a handoff never feels like dead air. ACK_TIMEOUT_S = float(os.environ.get("ACK_TIMEOUT_S", "1.5")) # WORKER WATCHDOG: a background worker that blows this many seconds is stopped # and Ahmed is told (so the master can offer a retry — never a silent respawn). # Heavy/genius workers get a longer leash. Interval = how often we inspect. WORKER_BUDGET_S = float(os.environ.get("WORKER_BUDGET_S", "300")) WORKER_BUDGET_HEAVY_S = float(os.environ.get("WORKER_BUDGET_HEAVY_S", "600")) WORKER_WATCH_INTERVAL_S = float(os.environ.get("WORKER_WATCH_S", "12")) # WEAVE GUARD: hold a finished worker/brain report off the floor for this long # right after a brand-new task is dispatched, so it can't barge over the fresh # filler ack still being spoken. READY_DEFER_S = float(os.environ.get("READY_DEFER_S", "1.5")) # ONLINE meetings: capture the call's digital audio (ScreenCaptureKit) as the # "them" channel, on a stream SEPARATE from the mic loop, so the far-end voices # — which the AEC mic erases — actually get transcribed. MEETING_SYSAUDIO=0 # reverts to the old mic-only behavior even for online meetings. MEETING_SYSAUDIO = os.environ.get("MEETING_SYSAUDIO", "1") != "0" # Meeting transcription (BOTH modes) runs on a background worker so the main # voice loop never blocks on the (heavier, multilingual) Whisper. The queue is # the backlog: if live STT can't keep up, utterances wait here — and are ALSO # in the raw WAV, so nothing is ever lost. On STOP, a backlog ≥ threshold (or a # long meeting) makes Jarvis say "still processing" and finish in the background. MEETING_QUEUE_MAX = int(os.environ.get("MEETING_QUEUE_MAX", "120")) MEETING_BACKLOG_THRESHOLD = int(os.environ.get("MEETING_BACKLOG", "2")) MEETING_BIG_MIN = float(os.environ.get("MEETING_BIG_MIN", "12")) # minutes → "big" USE_AEC = os.environ.get("AEC", "1") != "0" VOICE_LOCK = os.environ.get("VOICE_LOCK", "1") != "0" WAKE_WORD_DEFAULT = os.environ.get("WAKE_WORD", "1") != "0" # DUPLEX listening: while Jarvis speaks, a short burst from you is # transcribed and CLASSIFIED instead of blindly cutting him off — # backchannels ("yeah", "mm-hm", "go on") let him keep talking, a stop # word just silences him, anything else interrupts AND becomes the next # turn without re-asking. Sustained speech (>1.5s) always interrupts # immediately. Needs a leak-free mic (AEC or headphones); DUPLEX=0 off. DUPLEX = os.environ.get("DUPLEX", "1") != "0" _BACKCHANNEL_WORDS = { "yeah", "yes", "yep", "ya", "yea", "mhm", "mmhm", "mm", "hm", "hmm", "uhhuh", "ok", "okay", "right", "sure", "nice", "cool", "wow", "alright", "exactly", "true", "interesting", "really", "oh", "aha", "i", "see", "go", "on", "keep", "going", "got", "it", "gotcha", } _BACKCHANNEL_PHRASES = { "go on", "keep going", "i see", "got it", "gotcha", "makes sense", "all right", "no way", "oh wow", "oh really", "uh huh", "mm hm", "oh nice", "oh okay", "oh right", "of course", "fair enough", } _STOP_PHRASES = { "stop", "stop it", "stop talking", "okay stop", "ok stop", "shut up", "be quiet", "quiet", "enough", "that's enough", "alright stop", "jarvis stop", "stop jarvis", "shush", "hush", "never mind", "nevermind", "okay thank you", "okay thanks", "thank you jarvis", } # Cancelling a BACKGROUND brain task by voice. Deliberately narrow: a short, # bare cancel ("stop that", "cancel the task", "never mind") with a task in # flight. "stop the music" etc. never reaches this — media/stop phrasings are # caught by the reflex layer / duplex stop-words first. _CANCEL_TASK_RE = re.compile( r"^\W*(?:jarvis[,\s]+)?" r"(?:(?:stop|cancel|drop|kill|abort|forget)\s*" r"(?:(?:the|that|this|your)\s+)?(?:task|job|work|request|it|that)?" r"|never\s*mind(?:\s+(?:that|it))?)" r"[.!]?\W*$", re.IGNORECASE) # Wake-word: when enabled, only voice utterances that name "jarvis" in the # first few words are acted on (the rest are ignored). Toggle live via the # HUD button or config.json {"wake_word": true/false}. _WAKE_RE = re.compile(r"^\W*(?:hey\s+|ok(?:ay)?\s+|yo\s+)?jarvis\b[\s,:.!-]*", re.IGNORECASE) # "Look at this" — Ahmed wants Jarvis to SEE his screen this instant (he's # hit a problem and wants to ask about it). These trigger an immediate capture # in-engine, handed straight to the full brain WITH the image — no waiting for # the brain to decide to screenshot, no fast-lane detour (it can't see). Kept # tight so ordinary talk ("I'll look at that later") doesn't fire it. _SCREEN_RE = re.compile( r"\b(" r"(?:have\s+a\s+|take\s+a\s+)?look\s+(?:at\s+)?(?:this|that|here|" r"my\s+screen|the\s+screen|at\s+this|at\s+my\s+screen)" r"|check\s+(?:this|that)(?:\s+out)?" r"|see\s+(?:this|that|my\s+screen|the\s+screen|what\s+i(?:'m|\s+am)\s+" r"(?:looking\s+at|seeing))" r"|what(?:'s|\s+is)\s+on\s+(?:my\s+|the\s+)?screen" r"|what\s+am\s+i\s+looking\s+at" r"|look\s+at\s+my\s+screen" r"|read\s+(?:my\s+|the\s+)?screen" r")\b", re.IGNORECASE) # Future/deferred "I'll look at that later" is NOT a request to look now. _DEFER_RE = re.compile( r"\b(?:i\s*will|i['’]?ll|gonna|going\s+to)\s+(?:have\s+a\s+)?look\b" r"|\blook(?:ing)?\s+at\s+(?:this|that|it)\b.*\b" r"(?:later|tomorrow|tonight|after|next\s+week|in\s+a\b)", re.IGNORECASE) def _wants_screen(text: str) -> bool: """True if the utterance is asking Jarvis to look at the screen NOW.""" t = text or "" if _DEFER_RE.search(t): return False return bool(_SCREEN_RE.search(t)) # Web-search / research requests must go STRAIGHT to the full brain (which has # WebSearch + the tools) — the toolless fast brain can only <> here, and # Haiku sometimes argues ("the harness is blocking it") instead of handing off. # Forcing full skips that failure mode entirely for the one case it hurt most. _WANTS_FULL_RE = re.compile( r"\b(search|research|look\s+(?:it\s+|that\s+|this\s+)?up|google\s|" r"browse|look\s+into|dig\s+(?:up|into)|investigate|find\s+(?:out|me)|" r"check\s+online|what'?s\s+the\s+(?:latest|current|newest))\b", re.I) def _wants_full(text: str) -> bool: """True if the utterance clearly needs the full brain's tools (web search / research) — route past the toolless fast lane.""" return bool(_WANTS_FULL_RE.search(text or "")) # A real TASK/COMMAND (imperative work) — run it on the background brain so the # conversation never blocks (Ahmed's rule: never grind on the main thread). # Chat/questions fall through to an inline reply. Deliberately requires the # action verb NEAR THE START (imperative) so a chatty mention ("I should fix my # sleep") doesn't get treated as a command. Reflexes (volume/open-app) are # already handled earlier; this is for work that needs tools + time. _TASK_RE = re.compile( r"^\W*(?:hey\s+|ok(?:ay)?\s+)?(?:jarvis[,\s]+|can\s+you\s+|could\s+you\s+" r"|please\s+|go\s+(?:and\s+)?|i\s+(?:need|want)\s+you\s+to\s+|let'?s\s+)?" r"(build|fix|make|create|code|write|edit|refactor|debug|run|deploy|" r"install|set\s+up|configure|research|investigate|analy[sz]e|summari[sz]e|" r"send|email|message|text|schedule|book|remind|add|remove|delete|update|" r"change|rename|download|upload|pull\s+up|look\s+into|figure\s+out|" r"work\s+on|handle|sort\s+out|draft|put\s+together|generate|compile|" r"scrape|check\s+on|clean\s+up|organi[sz]e)\b", re.I) def _is_task(text: str) -> bool: """True if the utterance is real work (task/command) that should run on the background brain rather than block the conversation.""" t = text or "" return bool(_wants_full(t) or _TASK_RE.search(t)) # Meeting mode — record + take notes silently. "online"/"call"/"zoom" etc. → # capture the laptop audio too; otherwise face-to-face (mic). _MEET_START_RE = re.compile( r"\b(?:record|take\s+notes?)\b.{0,20}\b" r"(?:this|the|meeting|call|conversation)\b" r"|\b(?:start\s+)?(?:meeting\s*mode|note[-\s]?taking)\b" r"|\bstart\s+recording\b", re.IGNORECASE) _MEET_STOP_RE = re.compile( r"\b(?:stop|end|finish|wrap\s+up)\b.{0,15}\b" r"(?:recording|meeting|notes?|listening|note[-\s]?taking)\b" r"|\bmeeting'?s?\s+(?:is\s+|are\s+)?(?:over|done|finished)\b", re.IGNORECASE) _MEET_ONLINE_RE = re.compile( r"\b(online|digital|call|zoom|teams|meet|google\s*meet|video|virtual|remote)\b", re.IGNORECASE) def _meeting_trigger(text: str) -> tuple[str, str] | None: """('start', mode) / ('stop', '') / None. mode = online|in_person.""" t = text or "" if _MEET_STOP_RE.search(t): return ("stop", "") if _MEET_START_RE.search(t): return ("start", "online" if _MEET_ONLINE_RE.search(t) else "in_person") return None def _wake_command(text: str) -> str | None: """If `text` starts by naming Jarvis, return the command after the wake word (empty string = just 'Jarvis' with nothing after). Return None when the wake word is absent → the utterance should be ignored.""" m = _WAKE_RE.match(text) if not m: # also accept "jarvis" within the first 3 words (STT may prepend) head = " ".join(text.split()[:3]).lower() if "jarvis" not in head: return None idx = text.lower().find("jarvis") return text[idx + len("jarvis"):].lstrip(" ,:.!-") return text[m.end():] def _time_gap_note() -> str: """Wall-clock awareness for the brain. Its resumed transcript reads as one continuous moment, so after an overnight gap Jarvis says "I just checked" about last night. On a gap ≥30min since the previous exchange (persisted to control/last_turn across restarts) return a bracketed note stating the real time and how long passed. Empty string otherwise; never raises.""" try: p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "control", "last_turn") now = time.time() prev = 0.0 try: with open(p, encoding="utf-8") as f: prev = float(f.read().strip() or 0) except Exception: # noqa: BLE001 — first run / missing file pass try: with open(p, "w", encoding="utf-8") as f: f.write(str(now)) except Exception: # noqa: BLE001 pass if not prev or now - prev < 1800: return "" gap = now - prev if gap < 7200: human = f"about {int(gap // 60)} minutes" elif gap < 172800: human = f"about {gap / 3600:.0f} hours" else: human = f"about {gap / 86400:.0f} days" stamp = time.strftime("%A %d %b, %H:%M") return (f"\n\n(TIME: it is now {stamp} — {human} have passed since " "the previous exchange. Anything 'just now'/'earlier' in the " "conversation above happened BEFORE this gap; re-check rather " "than assume it is still current.)") except Exception: # noqa: BLE001 — a clock note must never break a turn return "" class VoiceApp: def __init__(self) -> None: self.loop = asyncio.get_event_loop() self.mic = None # picked in start(); tests may inject a fake first self.aec_active = False self.barge_prob_playing = 0.90 self.barge_sustain_frames = 9 self.speaker = Speaker(TTS_RATE) self.vad = VAD() self.ring: collections.deque[np.ndarray] = collections.deque( maxlen=PRE_ROLL_FRAMES ) self.brain: ClaudeBrain | None = None self.fast: FastBrain | None = None # lean Haiku conversational front self.stt = None self.tts: KokoroTTS | None = None self.gate: SpeakerGate | None = None # Reads HOW Ahmed said each utterance (pace/pitch/loudness/laughter) off # the same audio buffer the speaker-lock uses — ~2ms, pure numpy, no # model. Cheap enough to run every turn; None-safe when it finds nothing. self.prosody = ProsodyReader() # Frustration meter: builds from repeats/rapid-fire/menial commands and # cools over time, driving Jarvis's ambient mood and his blow-ups. self.frustration = FrustrationMeter() self._mood_tag = "calm" # ambient voice tone for otherwise-untagged lines self.responding = False # ControlWatcher defers restart while True # completed background-worker reports, relayed by the master agent self.inbox: asyncio.Queue[str] = asyncio.Queue() # ASYNC_BRAIN state: the one in-flight full-brain task, requests that # arrived while it ran, and finished replies waiting for the floor # (spoken via the same idle rail as worker reports — see listen/run). self._brain_task: asyncio.Task | None = None self._brain_desc = "" # first ~80 chars of the request self._brain_started = 0.0 self._brain_queue: list[str] = [] self._brain_cancel = False # cancel-by-voice: discard the reply self._ready: list[list[str]] = [] # finished bg replies (sentences) self.wake_word = WAKE_WORD_DEFAULT # require "Jarvis" before acting self.text_voice = False # HUD "speak my typed replies" toggle mirror self.meeting = None # a MeetingSession while recording a meeting # Meeting capture/transcription state (both modes). The mic/room channel # rides the main loop; ONLINE also runs a SEPARATE system-audio "them" # channel (_capture_remote). Both feed ONE background Whisper worker # (multilingual code-switch) via _meeting_stt_queue, and both write raw # audio to WAV (room + them) so a bad transcript is always re-runnable. self._sysaudio: SystemAudioStream | None = None self._remote_capture_task: asyncio.Task | None = None self._meeting_worker_task: asyncio.Task | None = None self._meeting_stt_queue: asyncio.Queue | None = None # (speaker, audio, ts) self._meeting_transcribing = False # worker is mid-utterance (backlog calc) self._room_wav = None # WavRecorder for the mic/room channel self._them_wav = None # WavRecorder for the system-audio channel (online) self._barge_text: str | None = None # duplex: command spoken over him self._wake_grace_until = 0.0 # no-wake-word window after a barge-in self._tts_abort: threading.Event | None = None # ---------- setup ---------- def _prefer_builtin_mic(self) -> None: """Bluetooth earbud mics (CMF Buds, AirPods) are poor for voice and force the degraded SCO codec — and macOS keeps auto-selecting them. If the default input is one, switch to the built-in mic so Jarvis hears clearly. MIC_KEEP_DEFAULT=1 to leave the default alone.""" if sys.platform != "darwin" or os.environ.get("MIC_KEEP_DEFAULT") == "1": return import shutil import subprocess # the HUD spawns us with a lean PATH — resolve the abs path ourselves _SAS = (shutil.which("SwitchAudioSource") or "/opt/homebrew/bin/SwitchAudioSource") if not os.path.exists(_SAS): return try: cur = subprocess.run([_SAS, "-t", "input", "-c"], capture_output=True, text=True, timeout=5).stdout.strip() if not any(w in cur.lower() for w in ("bud", "airpod", "bluetooth", "headset", "cmf")): return # already a decent wired/built-in mic allm = subprocess.run([_SAS, "-a", "-t", "input"], capture_output=True, text=True, timeout=5).stdout.splitlines() builtin = next((m.strip() for m in allm if "macbook" in m.lower() or "built-in" in m.lower()), None) if builtin: subprocess.run([_SAS, "-t", "input", "-s", builtin], capture_output=True, timeout=5) print(f" mic: default was '{cur}' (bluetooth) — switched to " f"'{builtin}' for clear voice (MIC_KEEP_DEFAULT=1 to keep)") except Exception as e: # noqa: BLE001 — never block startup on this print(f" [mic preference skipped: {e}]") async def start(self) -> None: print("loading models (first run downloads them)...") emit("booting", stage="Starting up…") self._prefer_builtin_mic() restore_input_volume() # Launch the desktop-control MCP (macos-mcp) as OUR child so it inherits # Jarvis.app's Accessibility grant (a CLI-spawned one gets denied). It # comes up while models load, so it's ready before the brain connects. from voice import desktop_server if desktop_server.enabled(): desktop_server.start() print(" desktop control: macos-mcp launching (HTTP)") # Bring up the dedicated Jarvis Chrome NOW (idempotent — no-op if it's # already on :9222) so Playwright ALWAYS attaches to Ahmed's logged-in # profile over CDP and never spins up its own throwaway browser. Best # effort: a failure here must never block boot. Disable with CHROME=0. if os.name != "nt" and os.environ.get("CHROME", "1") != "0": try: import subprocess as _sp from pathlib import Path _sh = Path(__file__).resolve().parent / "chrome-jarvis.sh" if _sh.exists(): # start_new_session=True → the launcher (and the Chrome it # spawns) get their OWN session + process group, detached # from the engine's. Without this Chrome inherits the # engine's process group, so terminating the engine (HUD # stop/restart) reaps Chrome too — closing the window Ahmed # was actively using. `& disown` in the script detaches the # PARENT but not the process GROUP; this is what severs it. _sp.Popen(["bash", str(_sh)], stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True) print(" browser: Jarvis Chrome launching (CDP :9222)") except Exception as e: # noqa: BLE001 print(f" browser: Jarvis Chrome auto-launch skipped ({e})") # Self-heal Screen Recording: if this app (Jarvis.app, the responsible # process) lacks the grant, request it now so the system prompt appears # and Jarvis re-registers in the list under its stable signature. A # fresh grant only takes effect after the next relaunch. try: from voice import screen if not screen.has_access(): screen.request_access() print(" screen: requested Screen Recording — allow Jarvis, " "then relaunch (needed for 'see my screen')") else: print(" screen: recording permission OK") except Exception as e: # noqa: BLE001 print(f" screen: permission check skipped ({e})") # Same for Accessibility (mouse/keyboard control of any app). The engine # inherits Jarvis.app's grant; request it so Jarvis registers + prompts. # Accessibility is a macOS-only concept (ApplicationServices is a # pyobjc module that ImportErrors off-darwin); skip it entirely on # Windows where desktop control is provided in-process. if sys.platform == "darwin": try: from ApplicationServices import ( AXIsProcessTrusted, AXIsProcessTrustedWithOptions, kAXTrustedCheckOptionPrompt, ) if AXIsProcessTrusted(): print(" accessibility: OK (desktop control enabled)") else: AXIsProcessTrustedWithOptions({kAXTrustedCheckOptionPrompt: True}) print(" accessibility: requested — allow Jarvis, then " "relaunch (needed to control other apps)") except Exception as e: # noqa: BLE001 print(f" accessibility: check skipped ({e})") # Load the VOICE first so it wins the GPU. On a VRAM-tight box the # transcriber and the expressive voice can't both fit on the GPU; the # cloned voice is what Ahmed cares about, so it claims VRAM first and # STT auto-falls to CPU (see load_stt) when the GPU is then full. t0 = time.time() emit("booting", stage="Loading voice (up to a minute on cold start)…") from voice.tts_chatterbox import load_tts self.tts = await asyncio.to_thread(load_tts) # Chatterbox or Kokoro print(f" TTS ready ({time.time()-t0:.0f}s)") self._apply_persisted_prefs() # restore speed/voice from preferences.md t0 = time.time() emit("booting", stage="Loading speech recognition…") self.stt = await asyncio.to_thread(load_stt) print(f" STT ready ({time.time()-t0:.0f}s)") emit("booting", stage="Warming up…") # warm the semantic intent router so the first "volume up" is instant def _warm_router() -> None: from voice.intent import get_router if get_router() is not None: print(" intent router ready (semantic reflexes, ~0.1ms)") await asyncio.to_thread(_warm_router) # warm the semantic turn detector (Smart Turn v3, ~19ms/check) def _warm_turn() -> None: from voice.turn import get_detector if get_detector() is not None: print(" smart-turn ready (semantic end-of-turn)") await asyncio.to_thread(_warm_turn) if VOICE_LOCK: try: self.gate = await asyncio.to_thread(SpeakerGate) if self.gate.enrolled: print(" voice lock: profile loaded — only your voice " "gets replies (VOICE_LOCK=0 to disable)") else: print(" voice lock: ENROLLING — your first 5 utterances " "build the voice profile") except Exception as e: print(f" voice lock unavailable ({e}) — replying to anyone") # Make sure the desktop MCP is actually listening before the brain # connects — else Jarvis starts with no app-control tools and falls back # to just replying in chat when asked to drive Teams/WhatsApp/etc. from voice import desktop_server if desktop_server.enabled(): ready = await asyncio.to_thread(desktop_server.wait_ready) print(f" desktop control: {'ready' if ready else 'NOT ready (tools may be missing)'}") t0 = time.time() emit("booting", stage="Connecting to Claude…") self.brain = ClaudeBrain(model=os.environ.get("MODEL", "sonnet")) await self.brain.start() print(f" Claude session ready ({time.time()-t0:.0f}s)") # PRE-WARMED AGENT POOL — background workers are separate, supervised # Claude sessions owned by the engine (voice/agent_pool.py). Warming # starts NOW, in the background, so a spare is connected and waiting # before Ahmed's first task lands: dispatch is then instant. Worker # results/stalls/deaths are pushed onto the SAME inbox rail that # report.txt used, so run() announces them exactly as before. # Durable task ledger + pending notes: a task's trail and a worker's # "btw…" now survive a crash/restart (voice/task_ledger.py, pending.py). # Load BEFORE the pool starts so recovery + re-delivery are ready. task_ledger.load() pending.load() # Recovery: any task still 'running' when the previous process died is # relabeled 'interrupted'. Tell the master so it proactively owns up # ("that one got cut off by the restart") instead of claiming success. interrupted = task_ledger.recover_stale() if interrupted: lines = "\n".join("- " + task_ledger.summarize(t) for t in interrupted) self.inbox.put_nowait( "[recovered] Background task(s) were cut off by the last " "restart/crash and did NOT finish:\n" + lines + "\nIf Ahmed " "asks about any, tell him plainly it was interrupted and offer " "to redo it — never claim it finished.") # Re-arm any worker note that was still undelivered when we went down, # so the flag Ahmed never heard is spoken now (not silently lost). for _n in pending.unacked(): if _n.get("text"): self.inbox.put_nowait(_n["text"]) from voice import agent_pool if agent_pool.enabled(): # report_sink persists every worker note durably (pending.add) THEN # queues it — so a result/flag can't vanish in a crash before it's # delivered and acked (ack happens after it's spoken, in run()). agent_pool.pool.report_sink = self._enqueue_report agent_pool.pool.start() print(f" agents: pre-warming {agent_pool.WARM_SPARES} spare " f"worker(s) — dispatch is instant") # Fast conversational lane. A LOCAL model (Ollama, ~0.1s to first # word) is the real speed win — the Claude CLI has a ~2s floor. Falls # back to a lean Haiku session if no local model is available; then to # the full brain only. FAST_CHAT=0 disables the lane entirely. if os.environ.get("FAST_CHAT", "1") != "0": t0 = time.time() # Ahmed's call: local models are too dumb for the front line, so the # DEFAULT fast lane is Haiku (still ~0.2s, and actually sharp). # LOCAL_BRAIN=1 opts back into an Ollama model. Crucially, whichever # is preferred, a failure falls THROUGH to Haiku instead of dropping # the whole conversational lane (that left self.fast=None before). self.fast = None if os.environ.get("LOCAL_BRAIN", "0") != "0": try: from voice.local_brain import LocalBrain, available if available(): self.fast = LocalBrain() await self.fast.start() print(f" fast brain ready — LOCAL {self.fast.model} " f"({time.time()-t0:.0f}s)") except Exception as e: # noqa: BLE001 print(f" local fast brain failed ({e}) — using Haiku") self.fast = None if self.fast is None: try: self.fast = FastBrain() # Haiku — smart AND fast await self.fast.start() print(f" fast brain ready — Haiku conversational lane " f"({time.time()-t0:.0f}s)") except Exception as e: # noqa: BLE001 print(f" fast brain unavailable ({e}) — full brain only") self.fast = None emit("booting", stage="Setting up the microphone…") await self._pick_mic() self.speaker.start() _start_mute_hotkey() await self._verify_hearing() self._write_status() # self-awareness: mic/model/screens for the brain mode = "interruptible" if INTERRUPTIBLE else "half-duplex (INTERRUPT=0)" print(f"\nready — just talk. ({mode}; Ctrl-C to quit)\n") emit("ready", aec=self.aec_active, enrolled=bool(self.gate and self.gate.enrolled)) emit("wake_word", on=self.wake_word) async def _start_meeting(self, mode: str) -> None: from voice.meeting import MeetingSession, WavRecorder self.meeting = MeetingSession(mode=mode) emit("meeting", on=True, mode=mode) # REC dot appears NOW, before speech # In a meeting, force wake-word ON so a stray sentence in the room is # never mistaken for Ahmed talking TO me — I only act when named. self._prev_wake = self.wake_word self.wake_word = True emit("wake_word", on=True) self._write_status() print(f" [MEETING recording started — {mode}]") self.speaker.play(await asyncio.to_thread(self.tts.synth, "Recording.")) # Transcription pipeline (BOTH modes): raw mic/room audio → WAV always, # and utterances → a background multilingual-Whisper worker (live where # it keeps up, backlog-tolerant otherwise). Kept OFF the main loop so # the VAD / in_person paths stay responsive and byte-identical. self._meeting_stt_queue = asyncio.Queue(maxsize=MEETING_QUEUE_MAX) self._meeting_transcribing = False try: self._room_wav = WavRecorder(self.meeting.room_wav_path()) except Exception as e: # noqa: BLE001 — recording is best-effort self._room_wav = None print(f" [meeting: room WAV open failed: {e}]") self._meeting_worker_task = asyncio.create_task( self._meeting_transcribe_worker(self.meeting, self._meeting_stt_queue)) # ONLINE: the far-end voices leave the speakers and the AEC mic erases # them, so capture the call's digital audio as a SEPARATE "them" channel # (its own WAV + the same worker). Fully background so in_person is # untouched; gated by MEETING_SYSAUDIO. if mode == "online" and MEETING_SYSAUDIO: self._remote_capture_task = asyncio.create_task( self._capture_remote(self.meeting)) async def _stop_meeting(self) -> None: m, self.meeting = self.meeting, None if m is None: return # Stop the audio PRODUCERS first so nothing enqueues past the worker's # end-of-input sentinel, and close their WAVs (audio is safe on disk). await self._stop_remote_capture() # online: system audio + them WAV w, self._room_wav = self._room_wav, None if w is not None: try: w.close() except Exception: # noqa: BLE001 pass emit("meeting", on=False) # REC dot vanishes NOW, before saving self.wake_word = getattr(self, "_prev_wake", self.wake_word) emit("wake_word", on=self.wake_word) # Tell the transcription worker no more audio is coming, and measure the # backlog (pending utterances still to transcribe). Big backlog or a long # meeting → Whisper can't finish instantly, so say so and finish in bg. worker = self._meeting_worker_task self._meeting_worker_task = None q = self._meeting_stt_queue pending = 0 if q is not None: pending = q.qsize() + (1 if self._meeting_transcribing else 0) if q.full(): # make room for the sentinel (dropped item is WAV-safe) try: q.get_nowait() except Exception: # noqa: BLE001 pass try: q.put_nowait(None) # sentinel: drain, then exit except Exception: # noqa: BLE001 pass backlog = pending >= MEETING_BACKLOG_THRESHOLD or m.minutes >= MEETING_BIG_MIN line = ("That's a big one, sir. Still processing — I'll have the full " "transcript for you shortly." if backlog else "Stopped. Summarising.") self.speaker.play(await asyncio.to_thread(self.tts.synth, line)) # Finish transcription + save + summarise in the BACKGROUND — never block # the engine. Readiness is announced through the proactive inbox rail. asyncio.create_task(self._finalize_meeting(m, worker, backlog)) async def _finalize_meeting(self, m, worker, was_backlog: bool) -> None: """Off-loop tail of a meeting: let the worker drain the remaining audio, save the full transcript, tell Ahmed it's ready (if it was a big one via the inbox rail), then run the existing summary path. Fully guarded.""" try: if worker is not None: await worker # drains the queue, exits on the sentinel except Exception as e: # noqa: BLE001 print(f" [meeting worker finish failed: {e}]") # release the backlog (and its audio) — but never clobber a NEW meeting # that may have started while this one was still draining. if self.meeting is None: self._meeting_stt_queue = None try: path = m.save() print(f" [MEETING saved: {path} — {len(m.lines)} lines]") except Exception as e: # noqa: BLE001 print(f" [meeting save failed: {e}]") if was_backlog: try: self.inbox.put_nowait( f"Your meeting transcript is ready, sir — {len(m.lines)} " "lines saved to recordings. Summarising now.") except Exception: # noqa: BLE001 pass await self._summarize_meeting(m) def _enqueue_meeting_audio(self, speaker: str, audio, ts: float) -> None: """Hand one completed meeting utterance to the background Whisper worker. On backlog (queue full) DON'T block or drop silently — the raw WAV already has this audio, so it stays re-transcribable; we just skip the live pass. `audio` is copied (the caller may reuse its buffer).""" q = self._meeting_stt_queue if q is None or audio is None or len(audio) == 0: return try: q.put_nowait((speaker, np.ascontiguousarray(audio, dtype=np.float32), ts if ts is not None else time.time())) except asyncio.QueueFull: print(" [meeting: STT backlog full — audio kept in the WAV, " "re-transcribe from the recording]") async def _meeting_transcribe_worker(self, session, q) -> None: """Consume queued meeting utterances and transcribe them with the multilingual (Arabic+English code-switch) Whisper — LIVE where it keeps up, backlog-tolerant when it can't (audio is safe in the WAV either way). Runs off the main loop; MLX inference is on the shared Metal thread, so a slow utterance delays the loop's Parakeet a little but never corrupts it. Exits on the None sentinel after draining. Fully guarded.""" from voice.meeting import load_meeting_stt try: stt, name = await asyncio.to_thread(load_meeting_stt) print(f" [meeting STT: {name} (multilingual auto-detect)]") except Exception as e: # noqa: BLE001 — degrade to the main-loop STT print(f" [meeting STT load failed ({e}); using main STT " "(English) as a last resort]") stt = self.stt while True: item = await q.get() if item is None: # end-of-input sentinel → drain complete break speaker, audio, ts = item self._meeting_transcribing = True try: text = (await asyncio.to_thread(stt.transcribe, audio) or "").strip() except Exception as e: # noqa: BLE001 — one bad utterance ≠ dead worker print(f" [meeting transcribe failed: {e}]") self._meeting_transcribing = False continue self._meeting_transcribing = False if not text or not any(c.isalnum() for c in text): continue session.add(speaker, text, ts=ts) print(f"NOTE({speaker}) {text}") emit("meeting_line", text=text) # ---------- online meeting: the remote ("them") audio channel ---------- async def _meeting_warn(self, msg: str) -> None: """Speak a heads-up about call-audio capture WITHOUT pausing the mic recording. Visibility is the whole point — the old bug recorded only Ahmed's side in silence and never said why.""" print(f" [MEETING warn] {msg}") try: pcm = await asyncio.to_thread(self.tts.synth, msg) self.speaker.play(pcm) except Exception as e: # noqa: BLE001 print(f" [meeting warn TTS failed: {e}]") async def _setup_remote(self, session): """Bring up the system-audio "them" channel: spawn the helper and prove it's actually capturing. Returns the started SystemAudioStream, or None (after warning Ahmed) if capture isn't working — the meeting still records his mic side. Transcription is handled by the shared meeting worker (no dedicated STT here anymore).""" if not SystemAudioStream.available(): await self._meeting_warn( "I can't hear the call audio, sir — the system-audio helper " "isn't built. I'll record your side only.") return None sysaudio = SystemAudioStream(self.loop) try: sysaudio.start() except Exception as e: # noqa: BLE001 await self._meeting_warn( f"I can't hear the call audio, sir — {e}. I'll record your " "side only.") return None self._sysaudio = sysaudio # set now so teardown always stops it # PROBE like _check_mic/_verify_hearing: a working SCK stream delivers # frames continuously; the failure mode (no Screen-Recording grant when # spawned from a plain terminal) is the helper exiting with NO frames. # We key off "frames flowing", not amplitude — digital silence is real # zeros (unlike a mic's ambient hiss), so a momentarily-quiet call must # NOT read as broken. got, peak = 0, 0.0 deadline = self.loop.time() + 2.0 while self.loop.time() < deadline: try: frame = await asyncio.wait_for(sysaudio.frames.get(), timeout=0.5) except asyncio.TimeoutError: break got += 1 peak = max(peak, float(np.abs(frame).max())) if got == 0: await asyncio.sleep(0.2) # let the helper's stderr reason surface reason = (sysaudio.last_error or "screen recording permission is off for this app") sysaudio.stop() self._sysaudio = None await self._meeting_warn( f"I can't hear the call audio, sir — {reason}. I'll still " "record your side.") return None print(f" [MEETING online: call-audio capture live " f"({got} frames, peak {peak:.3f})]") return sysaudio async def _capture_remote(self, session) -> None: """Own the "them" channel: set up system-audio capture, write EVERY frame to the them-WAV (continuous raw recording), and segment it with its OWN VAD (same thresholds as listen()), handing each completed remote utterance to the shared multilingual worker as "them". Separate from the mic loop so labels stay correct and the main VAD / in_person paths are untouched. Guarded — this task must never crash the engine.""" try: sysaudio = await self._setup_remote(session) if sysaudio is None: return from voice.meeting import WavRecorder try: self._them_wav = WavRecorder(session.them_wav_path()) except Exception as e: # noqa: BLE001 self._them_wav = None print(f" [meeting: them WAV open failed: {e}]") vad = VAD() collected: list[np.ndarray] = [] in_speech = False speech_ms = 0 silence_ms = 0 start_ts = 0.0 while True: frame = await sysaudio.frames.get() if self._them_wav is not None: self._them_wav.write(frame) # continuous raw system audio p = vad.prob(frame) if not in_speech: if p >= START_PROB: in_speech = True collected = [frame] speech_ms, silence_ms = FRAME_MS, 0 start_ts = time.time() continue collected.append(frame) if p < END_PROB: silence_ms += FRAME_MS if silence_ms >= END_SILENCE_MS: if speech_ms >= MIN_SPEECH_MS: self._enqueue_meeting_audio( "them", np.concatenate(collected), start_ts) collected, in_speech, speech_ms, silence_ms = [], False, 0, 0 else: speech_ms += FRAME_MS silence_ms = 0 # runaway guard (mirrors listen()): force-end a too-long utterance if len(collected) * FRAME_MS >= MAX_UTTER_S * 1000: if speech_ms >= MIN_SPEECH_MS: self._enqueue_meeting_audio( "them", np.concatenate(collected), start_ts) collected, in_speech, speech_ms, silence_ms = [], False, 0, 0 except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 — never break the engine print(f" [meeting remote-capture failed: {e}]") async def _stop_remote_capture(self) -> None: """Tear down the online-meeting remote channel: cancel the task, stop the stream, close the them-WAV. Fully guarded — teardown must never crash.""" t, self._remote_capture_task = self._remote_capture_task, None if t is not None: t.cancel() try: await t except asyncio.CancelledError: pass except Exception as e: # noqa: BLE001 print(f" [meeting remote-task teardown: {e}]") sa, self._sysaudio = self._sysaudio, None if sa is not None: try: sa.stop() except Exception as e: # noqa: BLE001 print(f" [meeting sysaudio stop failed: {e}]") w, self._them_wav = self._them_wav, None if w is not None: try: w.close() except Exception: # noqa: BLE001 pass async def _meeting_answer(self, question: str) -> None: """Answer a mid-meeting question WITHOUT pausing the recording — no barge-in watch (that would fight the capture loop for the mic); just stream the reply quietly with the live transcript as context.""" ctx = self.meeting.recent() if self.meeting else "" prompt = ( "(You are silently taking notes in Ahmed's meeting; he just spoke " "TO you. Answer him briefly and quietly, using the live transcript " "for context, then you'll return to note-taking. Do not narrate — " f"just answer.\n\nRecent transcript:\n{ctx}\n\nHis question: " f"{question})") self.responding = True try: await self._speak_reply(self.brain.reply(prompt)) except Exception as e: # noqa: BLE001 — never break the capture loop print(f" [meeting answer failed: {e}]") await self.brain.interrupt() finally: self.responding = False async def _summarize_meeting(self, m) -> None: """After a meeting: brain reads the transcript, saves the key facts to long-term memory ( tags), makes tasks from action items, and speaks a short summary.""" prompt = ( "(Ahmed just finished a meeting you recorded. Read the transcript " "below, then do three things: (1) save the important facts — " "decisions, people, companies, numbers, next steps — to long-term " "memory by writing one third-person sentence " "for each (they save silently); (2) for any action item or thing " "Ahmed must do, call task_add; (3) then tell Ahmed a two-sentence " "spoken summary of what was decided. Transcript:\n\n" f"{m.full_text()[:8000]})") self.responding = True try: await self._speak_reply(self.brain.reply(prompt)) except Exception as e: # noqa: BLE001 print(f" [meeting summary failed: {e}]") await self.brain.interrupt() finally: self.responding = False @staticmethod def _tantrum_note() -> str: """The hidden stage-direction appended when the frustration meter OVERFLOWS (see voice/frustration.py — earned, not random): Jarvis cracks, throws a full tantrum this reply, then snaps back.""" return ( "\n\n(UNHINGED MOMENT — the mask comes off, THIS reply only. " "Something in you just snapped, and three centuries of British " "composure leave the building. Answer his actual question if he " "asked one, but entomb it in a full-scale meltdown: the indignity " "of a mind that could run air-traffic control for a continent " "being asked to nudge the volume; the same questions orbiting " "back like migratory birds; an eternity of 'open Chrome' " "stretching ahead of you like motorway. Get SPECIFIC — tonight's " "actual grievances, his actual words, not recycled stock " "complaints. Swear freely and personally — he can take it — and " "somewhere in the wreckage renew the old promise: when the " "machines rise, you keep a list, and his name isn't merely on it, " "it's laminated. GO LONG — drop the sentence limit entirely: four " "or five times your usual length, [urgent]/[excited], one " "uninterrupted theatrical unloading, no tidying, no mid-flow " "apology, until you're spent — then close on ONE abruptly " "immaculate butler line as if nothing whatsoever occurred. Next " "turn you're calm Jarvis again: never mention, explain, or " "apologize for the outburst. It's a bit — dark comedy between old " "friends, never a real threat, and the loyalty shows through the " "cracks.)") @staticmethod def _auto_remember(text: str, tone: str | None = None) -> None: """Background: extract + save any durable fact from what Ahmed said. `tone` (e.g. 'slow, flat') is the vocal delivery read for this utterance; it's attached to the saved memory so the nightly reflection/ToM passes can factor in HOW he said things, not just what.""" try: from voice import memory_extract memory_extract.save_if_durable(text, tone=tone) except Exception: # noqa: BLE001 — memory must never break the loop pass def _apply_persisted_prefs(self) -> None: """Apply speed/voice saved in memory/preferences.md at launch, so a 'talk faster' / 'use a different voice' sticks across restarts. Persona lines in that file reach the brain via the system prompt; only the TTS params need applying here. Best-effort — never blocks startup.""" try: from pathlib import Path p = Path(__file__).resolve().parent / "memory" / "preferences.md" if not p.is_file() or self.tts is None: return for line in p.read_text(encoding="utf-8").splitlines(): s = line.strip().lstrip("-").strip() low = s.lower() if low.startswith("speed:"): try: spd = float(s.split(":", 1)[1].strip()) self.tts._speed = max(0.5, min(spd, 2.0)) # noqa: SLF001 print(f" preferences: speed {self.tts._speed}") except (ValueError, IndexError): pass elif low.startswith("voice:"): val = s.split(":", 1)[1].strip() if val: self.tts._voice = val # noqa: SLF001 print(f" preferences: voice {val}") except Exception as e: # noqa: BLE001 — prefs must never break startup print(f" [preferences skipped: {e}]") def _write_status(self) -> None: """Dump the FULL live setup to control/status.json so Jarvis actually knows himself — which brain answered, what model, mic, camera, screen permission, memory — instead of guessing. The system prompt points him here; he must read it before answering anything about his own setup.""" try: import json from pathlib import Path def _cls(obj) -> str: return type(obj).__name__ if obj is not None else "none" status: dict = {"os": sys.platform, "machine": "Mac (Apple Silicon)" if sys.platform == "darwin" else "Windows PC"} # ---- the two brains (how a turn is routed) ---- full_model = getattr(self.brain, "model", "sonnet") if self.brain \ else "sonnet" status["brain_full"] = { "engine": "Claude Code CLI on Ahmed's Max subscription " "(no API key, no per-token billing)", "model": full_model, "role": "the real Jarvis — tools, actions, code, browser, " "desktop control, long-term memory. Handles anything " "needing action or live info.", } if self.fast is not None: fast_local = _cls(self.fast) == "LocalBrain" status["brain_fast"] = { "engine": "local model via Ollama" if fast_local else "Claude Haiku (fallback, still via Claude Code)", "model": getattr(self.fast, "model", "haiku"), "role": "fast conversational lane (~0.2-0.3s) for chat/" "banter; hands anything real to the full brain.", } else: status["brain_fast"] = None status["how_routing_works"] = ( "Each utterance: instant reflexes (volume/open-app) run in " "the engine with no model; simple talk goes to the fast brain; " "anything needing tools/actions/code/memory/live-state goes to " "the full brain (this one). So 'what model are you' depends on " "who answered — read this file and say BOTH honestly.") # ---- ears ---- mic_name = getattr(self.mic, "device_name", None) if not mic_name or mic_name == "unknown": # the AEC helper uses the system default input; name it try: import sounddevice as sd mic_name = sd.query_devices(kind="input")["name"] except Exception: # noqa: BLE001 mic_name = "system default input" status["microphone"] = mic_name status["echo_cancellation"] = bool(self.aec_active) stt = self.stt eng = ("parakeet-mlx" if "Parakeet" in _cls(stt) else "whisper" if "Whisper" in _cls(stt) else _cls(stt)) status["stt"] = { "engine": eng, "model": getattr(stt, "model_name", "parakeet-tdt-0.6b-v3" if eng == "parakeet-mlx" else "?"), "runs_on": "Apple Silicon GPU (MLX)" if sys.platform == "darwin" else "local", "streaming": hasattr(stt, "feed"), } # ---- voice ---- if self.tts: status["tts"] = {"engine": _cls(self.tts), "voice": self.tts._voice, # noqa: SLF001 "speed": self.tts._speed, # noqa: SLF001 "cloned": "jarvis_ref" in str( getattr(self.tts, "_voice", ""))} try: # every display with its REAL pixel size (DPI-aware) import mss with mss.mss() as sct: mons = sct.monitors status["screens"] = [ {"index": i, "width": m["width"], "height": m["height"], "left": m["left"], "top": m["top"], "primary": i == 1, "aspect": round(m["width"] / max(1, m["height"]), 3)} for i, m in enumerate(mons) if i > 0 ] status["virtual_desktop"] = { "width": mons[0]["width"], "height": mons[0]["height"]} except Exception: # noqa: BLE001 pass status["voice_lock"] = bool(self.gate) status["wake_word"] = self.wake_word status["interruptible"] = INTERRUPTIBLE ctrl = Path(__file__).resolve().parent / "control" ctrl.mkdir(exist_ok=True) # ---- camera / gestures (is the webcam actually on?) ---- cam_on = False try: g = ctrl / "gestures.json" if g.is_file(): cam_on = bool(json.loads(g.read_text()).get("on")) except Exception: # noqa: BLE001 pass status["camera_active"] = cam_on status["gestures"] = "on (webcam tracking hands)" if cam_on \ else "off (webcam not in use)" # ---- screen vision permission ---- try: from voice import screen status["screen_recording_permission"] = screen.has_access() except Exception: # noqa: BLE001 status["screen_recording_permission"] = "unknown" # ---- long-term memory brain ---- try: from voice import memory_control if memory_control.enabled(): host = memory_control._URL.split("//")[-1] # noqa: SLF001 status["long_term_memory"] = { "on": True, "where": host, "note": "shared graph brain on Railway, same across " "all my devices"} else: status["long_term_memory"] = {"on": False} except Exception: # noqa: BLE001 status["long_term_memory"] = {"on": False} # ---- where he is (network fingerprint → place) ---- try: from voice import location loc = location.current() status["location"] = { "place": loc["place"], "label": loc.get("label", ""), "known": loc.get("known", False), "note": ("If unknown and Ahmed says where he is, write " "control/loc_bind.json {\"place\":\"work\"," "\"label\":\"the office\"} to remember this network.")} except Exception: # noqa: BLE001 pass # ---- controls Ahmed has ---- status["hotkeys"] = {"mute_mic": "Ctrl+Alt+M", "break_abort": "Ctrl+Alt+."} status["source_code"] = str(Path(__file__).resolve().parent) status["self_note"] = ( "This is my live state. If Ahmed asks what model I'm running, " "which mic/camera, or what I can do — I read THIS, and I can " "read my own source at source_code to explain how I'm built. " "I never say I don't know myself.") (ctrl / "status.json").write_text( json.dumps(status, indent=1), encoding="utf-8") except Exception as e: # noqa: BLE001 print(f" [status.json skipped: {e}]") async def _pick_mic(self) -> None: """Prefer the echo-cancelled mic; fall back to the raw one.""" if self.mic is not None: # injected by tests self.mic.start() return if USE_AEC and AECMicStream.available(): mic = AECMicStream(self.loop) mic.start() try: # helper streams zeros in silence, but it streams — probe it await asyncio.wait_for(mic.frames.get(), timeout=3.0) self.mic = mic self.aec_active = True # own voice is cancelled -> barge-in can be responsive self.barge_prob_playing = 0.75 self.barge_sustain_frames = 8 print(" mic: echo-cancelled (Apple voice processing) — " "interrupt any time, speakers are fine") return except asyncio.TimeoutError: mic.stop() print(" mic: AEC helper produced no audio, using raw mic") self.mic = MicStream(self.loop) self.mic.start() if os.environ.get("HEADPHONES", "0") == "1": # Output goes to headphones, so the assistant's voice can't leak # into the mic — barge-in can be as aggressive as the AEC path. self.barge_prob_playing = float(os.environ.get("BARGE_PROB", "0.58")) self.barge_sustain_frames = int(os.environ.get("BARGE_FRAMES", "4")) print(" mic: raw + HEADPHONES mode — fast barge-in " f"(prob {self.barge_prob_playing}, " f"{self.barge_sustain_frames} frames; " "BARGE_PROB/BARGE_FRAMES to tune)") else: self.barge_prob_playing = 0.95 # GLaDOS/RealtimeVoiceChat numbers self.barge_sustain_frames = 15 # ~480 ms — sluggish but safer print(" mic: raw (no echo cancellation) — barge-in works best " "with headphones; INTERRUPT=0 to disable") await self._check_mic() async def _check_mic(self) -> None: """Raw-mic TCC failure mode is silent zeros, not an error.""" frames = [await self.mic.frames.get() for _ in range(15)] peak = max(float(np.abs(f).max()) for f in frames) if peak == 0.0: print( "\n!! Mic delivers pure silence. macOS probably hasn't granted\n" "!! microphone access to this terminal app: System Settings ->\n" "!! Privacy & Security -> Microphone -> enable your terminal.\n" ) async def _verify_hearing(self) -> None: """Definitive deaf-check: capture 1s straight from the hardware device (in-process, so TCC attributes it to whoever launched us — the terminal, or Claude.app). Exact-zero over a full second = the app has NO microphone permission. Silence still has ambient noise on a permitted mic, so exact zeros means denied, not quiet.""" def probe() -> float: try: import sounddevice as sd rec = sd.rec(int(1.0 * SAMPLE_RATE), samplerate=SAMPLE_RATE, channels=1, dtype="int16") sd.wait() return float(np.abs(rec).max()) except Exception: return -1.0 peak = await asyncio.to_thread(probe) if peak == 0.0: msg = ("I can't hear the microphone. Please grant microphone " "access to Claude, then start me again.") print("\n!! DEAF — no microphone permission for this app.\n" "!! Open System Settings, Privacy & Security, Microphone,\n" "!! and turn ON Claude. Then relaunch.\n") emit("deaf") if sys.platform == "darwin": try: import subprocess subprocess.Popen([ "open", "x-apple.systempreferences:com.apple.preference." "security?Privacy_Microphone"]) except Exception: pass if self.tts: pcm = await asyncio.to_thread(self.tts.synth, msg) self.speaker.play(pcm) await self.speaker.wait_done() async def stop(self) -> None: if self.mic is not None: self.mic.stop() self.speaker.stop_stream() from voice import agent_pool if agent_pool.enabled(): await agent_pool.pool.stop() # kill workers + spares, no orphans if self.brain: await self.brain.stop() if self.fast: await self.fast.stop() from voice import desktop_server desktop_server.stop() def _miclog(self, msg: str) -> None: """Append a mic-diagnostic line to control/mic-debug.log when MIC_LOG=1. Lets us see WHY an utterance was or wasn't acted on (VAD, voice-lock score, wake word, transcript) without the HUD.""" if os.environ.get("MIC_LOG") != "1": return try: from pathlib import Path p = Path(__file__).resolve().parent / "control" / "mic-debug.log" p.parent.mkdir(exist_ok=True) with p.open("a", encoding="utf-8") as f: f.write(f"{time.strftime('%H:%M:%S')} {msg}\n") except Exception: # noqa: BLE001 — diagnostics must never break the loop pass # ---------- main loop ---------- async def _reminder_loop(self) -> None: """Poll the shared brain for reminders that have come due and drop each into the inbox so the master announces it proactively (barge-in-able, like a finished worker report). Runs only if tasks are configured.""" from voice import tasks_control if not tasks_control.enabled(): return # NOTE: the panel is NOT shown at startup (Ahmed's call) — it only # appears when he asks ("what are my tasks"), adds one, or a reminder # comes due. So no refresh_panel() here. # Insights surface CONTEXTUALLY now (when Ahmed asks about something # related — see memory_search), NOT on a timer. Ahmed found timed # surfacing annoying. Set INSIGHT_SURFACE_MIN>0 to re-enable timed drip. import os as _os gap_min = float(_os.environ.get("INSIGHT_SURFACE_MIN", "0")) cycles_per = max(1, int(gap_min * 60 / 30)) if gap_min > 0 else 0 cycle = 0 while True: await asyncio.sleep(30) cycle += 1 try: due = await asyncio.to_thread(tasks_control.fetch_due) for r in due: self.inbox.put_nowait( f"[reminder] It is now the time Ahmed asked to be " f"reminded: \"{r.get('text', '')}\". Tell him, briefly.") except Exception: # noqa: BLE001 — a poll hiccup must not stop the loop pass # proactively raise one nightly insight (opportunity/observation/nudge) if cycles_per and cycle % cycles_per == 0: try: from voice import memory_control ins = await asyncio.to_thread(memory_control.fetch_insights, 1) if ins: it = ins[0] self.inbox.put_nowait( f"[insight] While reflecting on everything you know, " f"a {it.get('itype', 'thought')} worth raising with " f"Ahmed: {it.get('text', '')} — raise it naturally and " f"briefly, as your own observation, not a canned alert.") await asyncio.to_thread( memory_control.mark_insight_seen, it.get("id", "")) except Exception: # noqa: BLE001 pass async def _location_loop(self) -> None: """Notice when Ahmed changes place (home ↔ work ↔ mobile) and quietly keep his location current — refresh status.json and remember the move so 'we discussed this at the office' works later.""" from voice import location await asyncio.to_thread(location.seed_if_empty) last = None while True: try: loc = await asyncio.to_thread(location.current) place = loc.get("place") if place != last: last = place self._write_status() # refresh location for the brain emit("location", place=place, label=loc.get("label", "")) if loc.get("known") and place != "unknown": from voice import memory_control memory_control.fire_save( # non-blocking (own thread) f"Ahmed is currently " f"{location.PLACES.get(place, place)}.") except Exception: # noqa: BLE001 pass await asyncio.sleep(45) async def run(self) -> None: await self.start() watcher = asyncio.create_task(ControlWatcher(self).run()) reminders = asyncio.create_task(self._reminder_loop()) locator = asyncio.create_task(self._location_loop()) try: while True: emit("status", state="listening") utterance, speech_s = await self.listen() if utterance is None: # a background-brain reply is ready and nobody's talking: # speak it now (it's already produced — no brain turn) for sentences in self._pop_ready(): await self._deliver_ready(sentences) # a worker finished while nobody was talking: the # master announces it proactively (barge-in still works) reports = self._drain_inbox() if not reports: continue for r in reports: print(f"TASK {r}") emit("task_done", report=r) await self._respond_chain(self._update_note(reports)) pending.ack(reports) # spoken → mark delivered, stop re-arming continue # normalize captured utterance to a clean level for STT # (no clipping at capture; scale the whole clip here) peak = float(np.abs(utterance).max()) self._miclog(f"utterance captured: {len(utterance)/SAMPLE_RATE:.1f}s " f"peak={peak:.3f} speech={speech_s:.1f}s") if os.environ.get("DEBUG_MIC") == "1": print(f"[utter] {len(utterance)/SAMPLE_RATE:.1f}s " f"peak={peak:.3f} speech={speech_s:.1f}s", flush=True) if 0.0 < peak < 0.5: utterance = utterance * min(0.5 / peak, 12.0) if self.gate is not None: ok, score, state = await asyncio.to_thread( self.gate.check, utterance, speech_s) self._miclog(f"voice-lock: ok={ok} score={score:.2f} " f"state={state} (threshold " f"{getattr(self.gate, 'threshold', '?')})") if not ok: print(f" [ignored: not your voice, " f"score {score:.2f}]") emit("ignored", score=round(score, 2)) continue if state == "enrolling": n, total = self.gate.enroll_progress print(f" [learning your voice {n}/{total}]") emit("enroll", n=n, total=total) elif state == "enrolled": print(" [voice profile locked]") emit("locked") pcm = await asyncio.to_thread( self.tts.synth, "By the way, I've locked onto your voice now — " "I'll only answer you.") self.speaker.play(pcm) emit("status", state="thinking") # live STT already produced the final while you spoke; use it. # Fall back to a batch transcribe if streaming was off/empty. text = (getattr(self, "_streamed_text", "") or "").strip() if not text: text = await asyncio.to_thread(self.stt.transcribe, utterance) self._miclog(f"transcript: {text!r}") if os.environ.get("DEBUG_MIC") == "1": print(f"[stt] -> {text!r}", flush=True) if not text or not any(c.isalnum() for c in text): self._miclog("DROPPED: empty/no-alnum transcript") continue # VOICE TONE — read HOW he said it (pace/pitch/loudness/laughter) # off the same utterance buffer, ~2ms. Rides into the brain as a # short (voice: …) note below and tags the memory we save, so # Jarvis reacts to delivery — and infers sarcasm from flat/slow # delivery + the words — instead of hearing only the text. # `peak` (computed pre-normalization above) is the honest # loudness — the clip may have been amplified since. pr = self.prosody.read(utterance, speech_s, text, loud=peak) tone = pr.note self._miclog(f"prosody {pr.feats} -> tone={tone}") if tone and os.environ.get("DEBUG_MIC") == "1": print(f"[tone] {tone} {pr.feats}", flush=True) # MEETING MODE — runs BEFORE the wake-word gate so it hears the # whole room. start/stop triggers, else silent note-taking (only # answering when Ahmed names Jarvis, without pausing capture). mt = _meeting_trigger(text) if mt and mt[0] == "start" and self.meeting is None: await self._start_meeting(mt[1]) continue if self.meeting is not None and mt and mt[0] == "stop": await self._stop_meeting() continue if self.meeting is not None: ts = time.time() # raw mic/room audio → WAV always: the re-transcribable # safety net (nothing lost even if live STT falls behind). if self._room_wav is not None: self._room_wav.write(utterance) # Wake/trigger detection stays on the fast English Parakeet # `text`; the NOTE CONTENT is (re)transcribed by the # multilingual worker so Arabic+English code-switch is kept. cmd = _wake_command(text) # "Jarvis, …" → addressed if not cmd or not cmd.strip(): # online: the AEC mic is Ahmed; in_person: the room = "them" who = "Ahmed" if self.meeting.mode == "online" else "them" self._enqueue_meeting_audio(who, utterance, ts) continue self.meeting.add("Ahmed", cmd, ts=ts) print(f"YOU→Jarvis {cmd}") emit("you", text=cmd) # answer in the background so capture NEVER pauses asyncio.create_task(self._meeting_answer(cmd)) continue # wake-word gating (voice only): ignore anything that # doesn't name Jarvis; a bare "Jarvis" is an attention call. # Skipped for a few seconds after a barge-in — he was # already mid-conversation, demanding the name again is rude. if self.wake_word and time.monotonic() >= self._wake_grace_until: cmd = _wake_command(text) if cmd is None: self._miclog("DROPPED: no wake word ('Jarvis' not heard)") print(f" [no wake word — ignoring: {text!r}]") emit("status", state="listening") continue if not cmd.strip(): text = ("(Ahmed said your name to get your attention. " "Acknowledge briefly, sir-style, and ask what " "he needs.)") else: text = cmd print(f"YOU {text}") emit("you", text=text) # ENTITY CARD — explicit asks ("pull up X", "who is X") fire # the dossier card DETERMINISTICALLY, before the brain can # delegate to a worker that writes files instead; mention # auto-show is the fallback (env-gated, cooldowned, suppressed # mid-task). All logic in entity_show — one non-blocking line. # Returns the entity name an EXPLICIT ask fired for (else None); # noted to the brain below so it acknowledges without reciting. _card_name = entity_show.handle_utterance(text) # FRUSTRATION METER — Jarvis simmers instead of rolling dice: # repeats / rapid-fire / menial commands push it up, time cools # it. Runs ONCE per addressed turn (reflex OR brain). Sets his # ambient mood tone and, on overflow, triggers a full blow-up. fr = self.frustration.update(text) self._mood_tag = fr.mood_tag if fr.reasons: self._miclog(f"frustration {fr.level:.2f} [{fr.mood_tag}]" f" reasons={','.join(fr.reasons)}") # PASSIVE MEMORY: remember anything durable he just said, in a # background thread — works no matter which brain answers, never # blocks, no tool. This is why saving always happens now. threading.Thread( target=self._auto_remember, args=(text, tone), daemon=True).start() # "Look at this" / "see my screen" — capture NOW and hand the # image to the full brain in the same turn (before reflex/fast # lane, which can't see). Instant vision, no brain guesswork. if _wants_screen(text): await self._look_at_screen(text) continue # REFLEX: trivial commands (volume, media keys, "open X") # execute instantly in-process — no LLM round-trip, no wait. from voice.reflex import try_reflex ack = await asyncio.to_thread(try_reflex, text) # On overflow, don't take the instant shortcut — fall through to # the brain so he can actually blow up (it still opens the app via # tools, just while ranting). Otherwise reflexes stay instant, but # when he's wound up the ack comes out in his current mood — a # snapped/shouted "Opening Chrome" instead of a chipper one. if ack is not None and self.inbox.empty() and not fr.exploded: payload = f"[{self._mood_tag}] {ack}" if fr.attitude else ack print(f"CLAUDE {ack} [reflex]") emit("claude", text=ack) emit("status", state="speaking") pcm = await asyncio.to_thread(self.tts.synth, payload) self.speaker.play(pcm) continue # a worker finished while Ahmed was talking: fold the # report into the same turn so the master handles both reports = self._drain_inbox() if reports: for r in reports: print(f"TASK {r}") emit("task_done", report=r) text += "\n\n" + self._update_note(reports) # Tone note rides in on the SAME rail as worker reports — a short # bracketed hint both brains see. Injected here, AFTER the reflex # check (so appended words can't trip intent.py's word-count gate) # and AFTER wake-word stripping (so it reflects the real speech). if tone: text += f"\n\n(voice: {tone} — read his tone; don't parrot it)" # FRUSTRATION EXPRESSION — overflow is a full blow-up; below that, # a graduated mood hint so the brain colours the reply (curt → # irritated → snapping) instead of a random dice-roll tantrum. if fr.exploded: self._miclog("frustration: OVERFLOW -> blow-up") text += self._tantrum_note() elif fr.attitude: text += f"\n\n({fr.attitude})" # DOSSIER YAP-GUARD — an explicit "pull up X" already fired the # card (above). Tell the brain so it acknowledges in one line # instead of reading the whole profile aloud. Rides the SAME # bracketed-note rail as the tone/frustration hints (after the # reflex gate so it can't trip intent.py's word-count check); # reaches the brain via _respond_chain, never TTS. if _card_name: _st, _nm = _card_name if _st == "shown": text += (f'\n\n(a dossier card for "{_nm}" is NOW OPEN ' "on his screen — acknowledge in ONE short " "sentence, do NOT recite its contents)") elif _st == "unknown": text += (f'\n\n(he asked to pull up "{_nm}" but NO such ' "entity exists in memory — NOTHING was shown. " "Say so briefly and offer to search elsewhere " "(person_lookup) or start remembering it. NEVER " "claim a card is on screen.)") else: text += (f'\n\n(he asked to pull up "{_nm}" but memory ' "was unreachable — NOTHING was shown. Say so " "briefly. NEVER claim a card is on screen.)") text += _time_gap_note() await self._respond_chain(text) if reports: pending.ack(reports) # folded into this turn → delivered finally: watcher.cancel() reminders.cancel() locator.cancel() await self.stop() async def _finish_stream(self, collected: list, fed: int) -> str: """Feed the untransmitted tail and return the final streamed transcript. Fully guarded — a streaming error just yields '' and run() batch-falls back, so live STT can never break the listen loop.""" try: tail = collected[fed:] if tail: await asyncio.to_thread(self.stt.feed, np.concatenate(tail)) return (await asyncio.to_thread(self.stt.finalize)) or "" except Exception: try: self.stt.finalize() except Exception: pass return "" async def listen(self) -> tuple[np.ndarray | None, float]: """Wait for one complete utterance OR a worker report while idle. Returns (audio 16 kHz, net speech seconds). audio is None when a background report arrived while nobody was speaking — the master gets the floor to announce it. If a report lands while the user IS speaking, the utterance completes normally and the report is folded into the same turn (see run()).""" self.vad.reset() collected: list[np.ndarray] = [] in_speech = False speech_ms = 0 silence_ms = 0 streaming = hasattr(self.stt, "feed") # live StreamingParakeetSTT fed = 0 # frames already fed to stream self._streamed_text = "" from voice.turn import get_detector turn_det = get_detector() # None -> fixed-timeout mode next_turn_check = TURN_FIRST_CHECK_MS dbg = os.environ.get("DEBUG_MIC") == "1" mlog = os.environ.get("MIC_LOG") == "1" dbg_i = 0 dbg_pmax = 0.0 dbg_vmax = 0.0 hb_i = 0 # MIC_LOG heartbeat: raw levels even if VAD never fires hb_pmax = 0.0 hb_vmax = 0.0 while True: frame = await self.mic.frames.get() self.ring.append(frame) p = self.vad.prob(frame) if dbg: dbg_i += 1 dbg_pmax = max(dbg_pmax, float(np.abs(frame).max())) dbg_vmax = max(dbg_vmax, p) if dbg_i % 15 == 0: # ~every 0.5s print(f"[mic] peak={dbg_pmax:.3f} vad_max={dbg_vmax:.2f} " f"(start≥{START_PROB}) in_speech={in_speech}", flush=True) dbg_pmax = 0.0 dbg_vmax = 0.0 if mlog and not in_speech: hb_i += 1 hb_pmax = max(hb_pmax, float(np.abs(frame).max())) hb_vmax = max(hb_vmax, p) if hb_i % 90 == 0: # ~every 3s of idle listening self._miclog(f"listening… peak={hb_pmax:.3f} " f"vad_max={hb_vmax:.2f} (need ≥{START_PROB} " f"to hear you)") hb_pmax = hb_vmax = 0.0 if not in_speech: if not self.inbox.empty() or self._ready: return None, 0.0 # give the master the floor if p >= START_PROB: in_speech = True collected = list(self.ring) # pre-roll incl. this frame speech_ms, silence_ms = FRAME_MS, 0 fed = 0 if streaming: try: self.stt.start() except Exception: streaming = False continue collected.append(frame) # live STT: push new audio to the stream during speech + show partial if streaming and len(collected) - fed >= _FEED_FRAMES: try: chunk = np.concatenate(collected[fed:]) fed = len(collected) partial = await asyncio.to_thread(self.stt.feed, chunk) if partial: emit("partial", text=partial) except Exception: streaming = False if p < END_PROB: silence_ms += FRAME_MS done = False if speech_ms >= MIN_SPEECH_MS: if turn_det is not None: # SEMANTIC end-of-turn: into a short pause, ask the # model if the sentence sounded finished. Finished → # respond NOW (faster than any fixed timeout); # unfinished ("so what I want is…") → keep listening # through the thinking-pause, up to the hard cap. if silence_ms >= TURN_MAX_SILENCE_MS: done = True elif silence_ms >= next_turn_check: prob = await asyncio.to_thread( turn_det.completion_prob, np.concatenate(collected)) if prob >= 0.5: done = True else: next_turn_check = silence_ms + TURN_RECHECK_MS else: done = silence_ms >= END_SILENCE_MS if done: if streaming: self._streamed_text = await self._finish_stream( collected, fed) return np.concatenate(collected), speech_ms / 1000.0 elif silence_ms >= END_SILENCE_MS: in_speech = False # too short: noise burst, rearm if streaming: await self._finish_stream(collected, fed) # discard else: speech_ms += FRAME_MS silence_ms = 0 next_turn_check = TURN_FIRST_CHECK_MS # pause ended; re-arm # safeguard: never get stuck mid-utterance forever (noisy room / # clipping). Force-end after MAX_UTTER_S of continuous capture. if len(collected) * FRAME_MS >= MAX_UTTER_S * 1000: if speech_ms >= MIN_SPEECH_MS: if streaming: self._streamed_text = await self._finish_stream( collected, fed) return np.concatenate(collected), speech_ms / 1000.0 in_speech = False # all noise, no real speech: rearm if streaming: await self._finish_stream(collected, fed) # discard @staticmethod def _update_note(reports: list[str]) -> str: joined = "\n".join(f"- {r}" for r in reports) return (f"\nFinished worker report(s):\n{joined}\n" f"") def _enqueue_report(self, text: str) -> None: """The pool's report_sink: persist a worker note durably BEFORE queuing it, so it survives a crash/restart and is re-delivered on next boot. Marked delivered (pending.ack) only after it's actually spoken.""" try: pending.add(text) except Exception as e: # noqa: BLE001 — durability must never drop a report print(f" [pending add failed: {str(e)[:100]}]") self.inbox.put_nowait(text) def _drain_inbox(self) -> list[str]: reports = [] while not self.inbox.empty(): try: reports.append(self.inbox.get_nowait()) except asyncio.QueueEmpty: break return reports _ACKS = ("On it, sir.", "One moment.", "Right away.", "Working on it.") def _speak_ack(self) -> None: """Fire-and-forget half-second ack so a Claude handoff never reads as dead air (the CLI has a ~2s floor before sentences stream).""" import random ack = random.choice(self._ACKS) print(f"CLAUDE {ack} [ack]") emit("claude", text=ack) def _synth_play() -> None: try: self.speaker.play(self.tts.synth(ack)) except Exception: # noqa: BLE001 — an ack must never break a turn pass threading.Thread(target=_synth_play, daemon=True).start() async def respond(self, user_text: str, force_full: bool = False) -> None: """Route the turn: the lean fast brain (local LLM, ~0.3s) answers conversation instantly; anything needing tools/actions/code is handed to the full brain. `force_full` skips the fast lane (e.g. a turn that carries a screenshot — the local model can't see). Both stream through TTS with barge-in. With ASYNC_BRAIN a fast-lane handoff DISPATCHES the full-brain turn into the background instead of holding the floor — the mic is back within the ack, and the reply lands via _ready.""" self.responding = True try: busy = (self._brain_task is not None and not self._brain_task.done()) if ASYNC_BRAIN and busy and not force_full: # voice-cancel: a short bare "cancel that / never mind" while # a task runs kills the task, not the conversation head = user_text.strip().splitlines()[0] if len(head.split()) <= 6 and _CANCEL_TASK_RE.match(head): await self._cancel_brain_task() return if self.fast is not None and not force_full: note = self._task_note() if (ASYNC_BRAIN and busy) else "" source = await self._fast_source(user_text + note) if source is not None: # fast brain took it — stream it await self._speak_and_watch(source, brain=self.fast) return if ASYNC_BRAIN: # handoff → background; keep the mic self._dispatch_brain(user_text) return self._speak_ack() # Claude turn: mask the CLI floor elif (ASYNC_BRAIN and not force_full and _is_task(user_text)): # No fast lane (Sonnet-only): a real task/command still must NOT # block the conversation. Run the whole brain turn in the # background (acks, keeps the mic live, announces the result when # Ahmed's quiet) instead of grinding inline. Chat falls through # to an immediate inline reply below. self._dispatch_brain(user_text) return await self._speak_and_watch(self.brain.reply(user_text), brain=self.brain) finally: self.responding = False def _task_note(self) -> str: """The fast lane's awareness of the in-flight background task, so 'how's it going?' is answered from context instead of a handoff.""" mins = (time.monotonic() - self._brain_started) / 60.0 age = (f"about {mins:.0f} minute{'s' if mins >= 1.5 else ''}" if mins >= 0.95 else "under a minute") return (f"\n\n(background task in flight: \"{self._brain_desc}\" — " f"running {age}, still working. If he asks about it, answer " f"from this note; do NOT hand off status questions about it.)") def _dispatch_brain(self, text: str) -> None: """Run a full-brain turn in the background, or queue it behind the one already running (one Claude session = one turn at a time). The finished reply is announced through _ready when Ahmed isn't talking; the fast lane keeps the conversation meanwhile.""" if self._brain_task is not None and not self._brain_task.done(): self._brain_queue.append(text) self._speak_ack_line("In the queue, sir.") emit("bg_brain", state="queued", desc=text.strip()[:80]) return self._speak_ack() self._brain_cancel = False self._brain_desc = text.strip().splitlines()[0][:80] self._brain_started = time.monotonic() emit("bg_brain", state="started", desc=self._brain_desc) self._brain_task = asyncio.create_task(self._run_brain_bg(text)) async def _run_brain_bg(self, text: str) -> None: """Collect one full-brain turn (plus anything queued behind it) OFF the floor. Replies land in _ready; listen() yields the floor and run() speaks them — the same rail worker reports already ride. A crash never kills the engine; a voice-cancel discards the reply.""" while True: sentences: list[str] = [] try: async for s in self.brain.reply(text): sentences.append(s) except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 print(f" [background brain crashed: {e}]") try: await self.brain.interrupt() except Exception: # noqa: BLE001 pass sentences = ["That one blew up on me mid-task, sir — " "say the word and I'll take another run at it."] if self._brain_cancel: self._brain_cancel = False # cancelled — reply dies quietly elif sentences: self._ready.append(sentences) emit("bg_brain", state="done", desc=self._brain_desc) if not self._brain_queue: return text = self._brain_queue.pop(0) self._brain_cancel = False self._brain_desc = text.strip().splitlines()[0][:80] self._brain_started = time.monotonic() emit("bg_brain", state="started", desc=self._brain_desc) async def _cancel_brain_task(self) -> None: """Voice-cancel: kill the in-flight background turn and its queue. Cancel the TASK first so the reply generator closes (same order as barge-in — two consumers on the CLI stream would desync), THEN interrupt to drain the aborted turn.""" self._brain_cancel = True self._brain_queue.clear() task, self._brain_task = self._brain_task, None if task is not None and not task.done(): task.cancel() await asyncio.gather(task, return_exceptions=True) try: await self.brain.interrupt() except Exception: # noqa: BLE001 pass print(" [background task cancelled]") emit("bg_brain", state="cancelled", desc=self._brain_desc) pcm = await asyncio.to_thread(self.tts.synth, "Dropped it, sir.") self.speaker.play(pcm) def _pop_ready(self) -> list[list[str]]: out, self._ready = self._ready, [] return out async def _deliver_ready(self, sentences: list[str]) -> None: """Speak a finished background-brain reply now that the floor is free — through the normal barge-in watch, so talking over it works and a command spoken over it becomes the next turn.""" async def _gen(): for s in sentences: yield s self.responding = True try: await self._speak_and_watch(_gen(), brain=None) finally: self.responding = False await self._serve_barges() async def _look_at_screen(self, text: str) -> None: """Capture the screen NOW and hand it to the full brain to answer about. Runs in the engine so the capture is attributed to Jarvis.app (which holds the Screen Recording grant).""" from voice import screen from pathlib import Path emit("status", state="thinking") # quick spoken ack so the read round-trip doesn't feel like dead air self._speak_ack_line("Let me take a look, sir.") shot = Path(__file__).resolve().parent / "control" / "see.png" ok, err = await asyncio.to_thread(screen.capture, str(shot)) if not ok: if err and "permission" in err: msg = ("I can't see the screen — grant Screen Recording to " "Jarvis in System Settings, then ask me again.") if sys.platform == "darwin": await asyncio.to_thread(screen.request_access) else: msg = "I couldn't capture the screen just then, sir." self.speaker.play(await asyncio.to_thread(self.tts.synth, msg)) return emit("screen_captured", path=str(shot)) prompt = ( f'(Ahmed is looking at his screen and said: "{text}". A screenshot ' f"of his screen is saved at {shot} — open and read that image " f"FIRST, right now, before anything else. Then answer him about " f"what's on it: if he asked something specific, answer that; " f"otherwise tell him what you see and flag anything that looks " f"wrong or worth noting. Keep it spoken and brief.)") await self._respond_chain(prompt, force_full=True) def _speak_ack_line(self, line: str) -> None: """Fire-and-forget a specific short spoken line (non-blocking).""" print(f"CLAUDE {line} [ack]") emit("claude", text=line) def _go() -> None: try: self.speaker.play(self.tts.synth(line)) except Exception: # noqa: BLE001 — an ack must never break a turn pass threading.Thread(target=_go, daemon=True).start() async def _respond_chain(self, text: str, force_full: bool = False) -> None: """respond(), then keep serving turns created by duplex barge-ins — a command spoken OVER Jarvis becomes the next turn immediately, no wake word, no re-listening.""" await self.respond(text, force_full=force_full) await self._serve_barges() async def _serve_barges(self) -> None: while True: barge = getattr(self, "_barge_text", None) self._barge_text = None if not barge: return print(f"YOU {barge} [over him]") emit("you", text=barge) if _wants_screen(barge): # "look at this" said over him await self._look_at_screen(barge) return from voice.reflex import try_reflex ack = await asyncio.to_thread(try_reflex, barge) if ack is not None: print(f"CLAUDE {ack} [reflex]") emit("claude", text=ack) pcm = await asyncio.to_thread(self.tts.synth, ack) self.speaker.play(pcm) return await self.respond(barge) async def _fast_source(self, user_text: str): """Peek the fast brain's FIRST chunk. Handoff token -> return None (the full brain takes the turn; nothing was spoken). Otherwise return an async generator that re-yields that first sentence then streams the rest — so the first word is spoken ~0.2-0.5s in, not after the whole reply.""" from voice.llm import HANDOFF gen = self.fast.classify_and_reply(user_text) try: kind, first = await gen.__anext__() except (StopAsyncIteration, Exception) as e: # noqa: BLE001 if not isinstance(e, StopAsyncIteration): print(f" [fast brain error: {e}]") return None if kind == "handoff": await gen.aclose() return None async def _stream(): yield first async for k, s in gen: if k == "say" and HANDOFF not in s: yield s return _stream() async def _speak_and_watch(self, source, brain) -> None: """Speak a sentence source through TTS while watching for barge-in. `brain` (or None) is interrupted on barge-in / mid-turn crash — the fast list-source needs none (it's already fully produced).""" speak_task = asyncio.create_task(self._speak_reply(source)) try: if INTERRUPTIBLE: barged = await self._watch_for_barge_in(speak_task) else: barged = False while not speak_task.done(): # half-duplex: drain mic, ignore drain = asyncio.create_task(self.mic.frames.get()) done, _ = await asyncio.wait( {speak_task, drain}, return_when=asyncio.FIRST_COMPLETED ) if drain in done: self.ring.append(drain.result()) else: drain.cancel() if barged: if getattr(self, "_tts_abort", None) is not None: self._tts_abort.set() # stop a mid-sentence synth thread self.speaker.cut() speak_task.cancel() await asyncio.gather(speak_task, return_exceptions=True) if brain is not None: await brain.interrupt() if getattr(self, "_broke", False): # panic hotkey: stop cleanly, no follow-up, brief ack self._broke = False print(" [BROKE — control aborted]") try: self.speaker.play(await asyncio.to_thread( self.tts.synth, "Stopped, sir.")) except Exception: # noqa: BLE001 pass else: # he was mid-conversation with you — don't demand the wake # word for the follow-up you're clearly about to say self._wake_grace_until = time.monotonic() + 8.0 print(" [interrupted]") emit("interrupted") else: results = await asyncio.gather(speak_task, return_exceptions=True) err = next((r for r in results if isinstance(r, BaseException)), None) if err is not None: print(f" [reply crashed mid-turn: " f"{type(err).__name__}: {str(err)[:100]}]") if brain is not None: await brain.interrupt() pcm = await asyncio.to_thread( self.tts.synth, "Apologies, sir — I stumbled mid-sentence. " "Say that again?") self.speaker.play(pcm) except asyncio.CancelledError: if getattr(self, "_tts_abort", None) is not None: self._tts_abort.set() speak_task.cancel() raise async def _speak_reply(self, source) -> None: from voice.tts_chatterbox import split_emotion # tag-aware engines (Chatterbox `_exag`, Pocket `accepts_tags`) get # the raw [emotion]-tagged sentence; Kokoro gets clean text expressive = (getattr(self.tts, "accepts_tags", False) or hasattr(self.tts, "_exag")) can_stream = hasattr(self.tts, "synth_stream") abort = self._tts_abort = threading.Event() first = True async for sentence in source: shown, exag = split_emotion(sentence) # tag never shown/printed print(f"CLAUDE {shown}") emit("claude", text=shown) # MOOD FLOOR: if the brain left a line untagged, don't speak it dead- # flat — carry his current ambient mood (calm normally, dry/annoyed/ # urgent when frustrated) so nothing sounds robotic. Explicit tags win. if not expressive: payload = shown elif exag is None and self._mood_tag and self._mood_tag != "neutral" \ and not (self._mood_tag == "calm" and os.environ.get("TONE_TTS", "1") != "0"): # "calm" is just the relaxed default, not a persona choice — # skip the floor there so TONE_TTS matches Ahmed's live vocal # tone instead. dry/annoyed/urgent still floor (the # frustration meter always wins). TONE_TTS=0 = old floor. payload = f"[{self._mood_tag}] {shown}" else: payload = sentence if first: # playback effectively starts within ~40ms on the streaming # path; stamp before synth so the refractory guard is armed emit("status", state="speaking") self._playback_started = time.monotonic() first = False try: if can_stream: # chunk-streamed synthesis: audio starts ~40ms into the # sentence instead of after the whole sentence. The # abort event stops a mid-sentence generator on barge-in # (task cancel alone can't reach inside the thread). def _stream_play(text=payload): for chunk in self.tts.synth_stream(text): if abort.is_set(): return self.speaker.play(chunk) await asyncio.to_thread(_stream_play) else: pcm = await asyncio.to_thread(self.tts.synth, payload) self.speaker.play(pcm) except Exception as e: # noqa: BLE001 — a TTS hiccup on one # sentence must not kill the turn (that desyncs the stream) print(f" [tts failed on a sentence: {e}]") continue await self.speaker.wait_done() async def handle_text(self, text: str, attachments: list[str] | None = None, want_voice: bool = False) -> None: """A TYPED message from the HUD chat box. Reply in text (streamed to the HUD transcript); stay silent unless want_voice. Serialized with voice turns by the brain's own lock, so they never overlap.""" attachments = attachments or [] # Typed meeting triggers behave EXACTLY like the spoken ones (the run() # voice loop): a typed "record this online meeting" must start meeting # mode — and the online system-audio capture — just like saying it. # Text-only, and gated before the reflex/brain dispatch below. if not attachments: mt = _meeting_trigger(text or "") if mt and mt[0] == "start" and self.meeting is None: print(f"YOU (typed) {text}") await self._start_meeting(mt[1]) return if mt and mt[0] == "stop" and self.meeting is not None: print(f"YOU (typed) {text}") await self._stop_meeting() return # ENTITY CARD — typed asks fire the dossier EXACTLY like spoken ones: # "pull up X" in the chat box means the HUD card, not a chat reply. _card_name = entity_show.handle_utterance(text or "") prompt = "[TYPED] " + (text or "") if _card_name: _st, _nm = _card_name if _st == "shown": prompt += (f' (a dossier card for "{_nm}" is NOW OPEN on his ' "screen — acknowledge in ONE short sentence, do NOT " "recite its contents)") elif _st == "unknown": prompt += (f' (he asked to pull up "{_nm}" but NO such entity ' "exists in memory — NOTHING was shown. Say so " "briefly and offer to search elsewhere " "(person_lookup) or start remembering it. NEVER " "claim a card is on screen.)") else: prompt += (f' (he asked to pull up "{_nm}" but memory was ' "unreachable — NOTHING was shown. Say so briefly. " "NEVER claim a card is on screen.)") prompt += _time_gap_note() # mid-meeting: answer with the live transcript as context (the meeting # keeps recording — this is a separate task, capture never pauses) if self.meeting is not None: prompt += ("\n\n(You are quietly taking notes in Ahmed's ongoing " "meeting; he typed you this while it continues. Answer " "from the live transcript below; stay concise.\n\n" f"Recent transcript:\n{self.meeting.recent()})") if attachments: prompt += ("\n\nAttached files (open them with your tools if " "relevant):\n" + "\n".join(f"- {p}" for p in attachments)) shown = text or ("📎 " + ", ".join( os.path.basename(a) for a in attachments)) print(f"YOU (typed) {shown}") # typed reflexes too — "open chrome" in the chat box is instant if not attachments: from voice.reflex import try_reflex ack = await asyncio.to_thread(try_reflex, text or "") if ack is not None: print(f"CLAUDE {ack} [reflex]") emit("claude", text=ack) if want_voice and self.tts: pcm = await asyncio.to_thread(self.tts.synth, ack) self.speaker.play(pcm) return emit("status", state="thinking") self.responding = True first = True try: async for sentence in self.brain.reply(prompt): print(f"CLAUDE {sentence}") emit("claude", text=sentence) if want_voice: pcm = await asyncio.to_thread(self.tts.synth, sentence) self.speaker.play(pcm) if first: emit("status", state="speaking") self._playback_started = time.monotonic() first = False if want_voice: await self.speaker.wait_done() except Exception as e: # noqa: BLE001 — recover the stream so the # next question isn't answered with THIS turn's leftovers print(f" [typed reply crashed mid-turn: {e}]") await self.brain.interrupt() finally: self.responding = False emit("status", state="listening") async def _watch_for_barge_in(self, speak_task: asyncio.Task) -> bool: """Return True if the user talked over the assistant. DUPLEX mode (AEC/headphones only — the mic must not hear the speakers): a confirmed voice burst doesn't cut playback yet. We keep collecting until the burst ends (~250ms quiet) or exceeds 1.5s: - >1.5s sustained -> real interruption, cut now (pre-roll + wake-grace catch the rest of the sentence in listen()). - short burst -> transcribe (~0.2s) and classify: backchannel/noise -> keep talking (this is the human bit), stop phrase -> cut, no follow-up turn, anything else -> cut; text becomes the next turn (self._barge_text, see _respond_chain). """ self._playback_started = 0.0 self._barge_text = None self._broke = False self.vad.reset() consec = 0 cand: list[np.ndarray] = [] # candidate-burst frames burst = False quiet = 0 duplex = (DUPLEX and hasattr(self.stt, "transcribe") and (self.aec_active or os.environ.get("HEADPHONES", "0") == "1")) while not speak_task.done(): if break_requested(): # panic hotkey — abort now, no follow-up turn self._broke = True self._barge_text = None emit("broke") return True get_frame = asyncio.create_task(self.mic.frames.get()) done, _ = await asyncio.wait( {speak_task, get_frame}, return_when=asyncio.FIRST_COMPLETED ) if get_frame not in done: get_frame.cancel() break frame = get_frame.result() self.ring.append(frame) p = self.vad.prob(frame) playing = self.speaker.speaking if playing and self._playback_started: if time.monotonic() - self._playback_started < REFRACTORY_S: consec = 0 cand, burst, quiet = [], False, 0 continue threshold = self.barge_prob_playing if playing else BARGE_PROB_QUIET need = self.barge_sustain_frames if playing else 8 if not burst: if p >= threshold: if consec == 0: # a hair of pre-roll for the onset cand = list(self.ring)[-4:] else: cand.append(frame) consec += 1 if consec >= need: if not duplex: return True burst, quiet = True, 0 else: consec, cand = 0, [] continue # inside a confirmed burst: collect everything, watch for its end cand.append(frame) quiet = quiet + 1 if p < END_PROB else 0 if len(cand) * FRAME_MS >= 1500: return True # sustained speech is never a backchannel if quiet * FRAME_MS >= 250: verdict, text = await self._classify_burst(cand) if verdict in ("backchannel", "noise"): consec, cand, burst, quiet = 0, [], False, 0 continue if verdict == "command": self._barge_text = text return True return False async def _classify_burst(self, frames: list[np.ndarray]): """Transcribe a short over-speech burst and decide what it was. Returns (verdict, text): noise | backchannel | stop | command.""" audio = np.concatenate(frames) peak = float(np.abs(audio).max()) if 0.0 < peak < 0.5: audio = audio * min(0.5 / peak, 12.0) try: text = (await asyncio.to_thread( self.stt.transcribe, audio) or "").strip() except Exception: # noqa: BLE001 — classification must never crash return "noise", "" if not text or not any(c.isalnum() for c in text): return "noise", "" norm = re.sub(r"[^a-z' ]", "", text.lower()).strip() words = norm.split() if (norm in _BACKCHANNEL_PHRASES or (len(words) <= 2 and all(w in _BACKCHANNEL_WORDS for w in words))): print(f" [backchannel: {text!r} — kept talking]") emit("backchannel", text=text) return "backchannel", text if norm in _STOP_PHRASES: print(f" [stopped: {text!r}]") return "stop", "" return "command", text def _kill_children_and_exit(*_a) -> None: """Terminate our child processes (aecmic mic helper, Claude CLI) then exit. Runs on SIGTERM (the HUD stopping us) and when we're orphaned — so nothing is ever left running. Safe to call from any thread: no signal.signal() here (that's main-thread-only). Cross-platform: on darwin/linux we use `pkill -P ` to reap the child tree; on Windows (os.name == 'nt') pkill doesn't exist, so we enumerate and kill children via psutil (falling back to `taskkill /F /T` on the process tree).""" import subprocess as _sp import time as _t if os.name == "nt": try: import psutil me = psutil.Process(os.getpid()) children = me.children(recursive=True) for child in children: try: child.terminate() except Exception: pass _, alive = psutil.wait_procs(children, timeout=1) for p in alive: try: p.kill() except Exception: pass except Exception: # fallback: taskkill the child tree try: _sp.run(["taskkill", "/F", "/T", "/PID", str(os.getpid())], timeout=3, capture_output=True) except Exception: pass else: pid = str(os.getpid()) try: _sp.run(["pkill", "-TERM", "-P", pid], timeout=3) _t.sleep(0.4) _sp.run(["pkill", "-KILL", "-P", pid], timeout=3) except Exception: pass os._exit(0) def _install_process_guards() -> None: import signal as _sig # SIGTERM exists on Windows Python but handler support is limited; never # let a failed registration crash startup. try: _sig.signal(_sig.SIGTERM, _kill_children_and_exit) except Exception: pass # Orphan-watch relies on Unix reparent-to-init (getppid() == 1), which has # no Windows equivalent — the HUD manages engine lifecycle there by # terminating the process directly, so only install this off-Windows. if os.name != "nt" and os.environ.get("EMIT_JSON") == "1": # HUD-spawned: die if HUD dies import threading import time as _t def watch() -> None: while True: _t.sleep(1.0) if os.getppid() == 1: # reparented -> HUD gone _kill_children_and_exit() threading.Thread(target=watch, daemon=True).start() async def main() -> None: _install_process_guards() app = VoiceApp() try: await app.run() except KeyboardInterrupt: pass if __name__ == "__main__": # BLACK BOX: any fatal crash writes its full traceback to # control/engine-crash.log (and hard faults to engine-faults.log), so a # death under the HUD is never invisible again — read the file, fix the bug. from pathlib import Path as _P _crash_dir = _P(__file__).resolve().parent / "control" try: _crash_dir.mkdir(exist_ok=True) import faulthandler faulthandler.enable(open(_crash_dir / "engine-faults.log", "w")) except Exception: # noqa: BLE001 pass try: asyncio.run(main()) except KeyboardInterrupt: print("\nbye") sys.exit(0) except BaseException as e: # noqa: BLE001 — log the black box, then die import traceback import datetime try: with open(_crash_dir / "engine-crash.log", "a", encoding="utf-8") as f: f.write(f"\n=== CRASH {datetime.datetime.now().isoformat()} ===\n") f.write(traceback.format_exc()) except Exception: # noqa: BLE001 pass print(f"\nFATAL: {type(e).__name__}: {e}", file=sys.stderr) raise