A migration from a pre-PM venv carries the extras that venv held (matrix, fal, ...) into PM's first generation. Those reached sync_venv as if the user had requested them, so one that no longer builds (python-olm's vendored libolm without a working cmake/C++ compiler) failed the whole sync. No facts.json was written, the old venv is never deleted, and every later `hermes update` re-scanned it, re-carried the same extra and failed the same way: one optional extra bricked updates for good. sync_venv now takes `carried`, the entries of `extras` that were carried rather than asked for. When the sync fails with a build or resolver verdict, pm.carried_extras proves core plus the requested extras build without them, re-adds each carried extra on top of what fits, and syncs without the ones that do not. A build verdict is recorded in declined-extras.json (with the reason, so a later lazy-install refusal names the cause instead of a `--without` nobody ran) and the operator is told which extra was left out and to run `hermes pm install --extra NAME` to add it back. A download failure is left out of this update but not recorded. Requested extras and core still fail loudly. Both legacy callers (the source-launch sync and the historical takeover) pass the carried list. The streamed uv tail also keeps the build-backend marker now: a native build's traceback runs past the 2000-character tail, so the failure read as a fetch error instead of a BuildFailure verdict (this also hid plugin build verdicts from plugin eviction). Also fixes committed_venv's docstring: PM never deletes the in-tree venv.
405 lines
16 KiB
Python
405 lines
16 KiB
Python
"""Extras: the runtime features of the python venv.
|
|
|
|
Feature names ARE pyproject extra names. This table maps each extra to the
|
|
import that proves it (the anchor), so availability is one find_spec — no
|
|
package-manager machinery on the hot path. sync_venv([extra]) makes an extra
|
|
true; pm owns HOW (uv sync inside the venv package).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
|
|
# extra name -> module that proves it is installed
|
|
ANCHORS: dict[str, str | tuple[str, ...]] = {
|
|
"anthropic": "anthropic",
|
|
"bedrock": "boto3",
|
|
"vertex": "google.auth",
|
|
"azure-identity": "azure.identity",
|
|
"exa": "exa_py",
|
|
"firecrawl": "firecrawl",
|
|
"parallel-web": "parallel",
|
|
"ddgs": "ddgs",
|
|
"otlp": "opentelemetry.sdk",
|
|
"mistral": "mistralai",
|
|
"edge-tts": "edge_tts",
|
|
"neutts": "neutts",
|
|
"kittentts": ("kittentts", "soundfile"),
|
|
"piper": "piper",
|
|
"tts-premium": "elevenlabs",
|
|
"voice": "faster_whisper",
|
|
"stt-whisper": "faster_whisper",
|
|
"audio-io": ("sounddevice", "numpy"),
|
|
"silk": "pilk",
|
|
"wake": "pyopen_wakeword",
|
|
"wake-openwakeword": "pyopen_wakeword",
|
|
"wake-sherpa": "sherpa_onnx",
|
|
"wake-porcupine": "pvporcupine",
|
|
"fal": "fal_client",
|
|
"messaging": "telegram",
|
|
"telegram": "telegram",
|
|
"discord": "discord",
|
|
"slack": "slack_bolt",
|
|
"matrix": ("mautrix", "asyncpg", "aiosqlite", "markdown", "aiohttp_socks"),
|
|
"dingtalk": "dingtalk_stream",
|
|
"feishu": "lark_oapi",
|
|
"wecom": "defusedxml",
|
|
"keychain": "keyring",
|
|
"teams": "microsoft_teams.apps",
|
|
"modal": "modal",
|
|
"daytona": "daytona",
|
|
"vercel": "vercel",
|
|
"google": ("googleapiclient", "google.auth", "google_auth_oauthlib.flow", "google_auth_httplib2"),
|
|
"google-chat": "google.cloud.pubsub_v1",
|
|
"google-meet": ("playwright.sync_api", "websockets"),
|
|
"youtube": "youtube_transcript_api",
|
|
"acp": "acp",
|
|
"web": "fastapi",
|
|
"doc-extract": "anydoc",
|
|
"documents": ("docx", "openpyxl", "pptx", "reportlab", "pypdf"),
|
|
"documents-pdf": ("pdfplumber", "pypdfium2"),
|
|
"computer-use": "mcp",
|
|
"trace-upload": "huggingface_hub",
|
|
# Pillow resize recovery for vision tools (the `vision` extra is a no-op
|
|
# back-compat alias — Pillow is core — but ensure_import("vision") must
|
|
# still resolve an anchor so availability checks work).
|
|
"vision": "PIL",
|
|
}
|
|
|
|
|
|
def _anchors(extra: str) -> tuple[str, ...]:
|
|
got = ANCHORS.get(extra, extra.replace("-", "_"))
|
|
return got if isinstance(got, tuple) else (got,)
|
|
|
|
|
|
def _importable(anchor: str) -> bool:
|
|
"""An anchor already present in sys.modules counts even without a
|
|
findable spec — tests fake SDKs by inserting modules there, and the
|
|
caller's import right after this check resolves the same way."""
|
|
import sys
|
|
|
|
if anchor in sys.modules:
|
|
return True
|
|
try:
|
|
return importlib.util.find_spec(anchor) is not None
|
|
except (ImportError, ValueError):
|
|
return False
|
|
|
|
|
|
def available(extra: str) -> bool:
|
|
"""Fast, side-effect-free: are all of this extra's anchors importable?
|
|
A platform-gated extra whose gate excludes this machine reads as
|
|
unavailable — the anchors are absent by design, not by accident."""
|
|
if not extra_supported(extra):
|
|
return False
|
|
return all(_importable(a) for a in _anchors(extra))
|
|
|
|
|
|
def _platform_gates() -> dict[str, str]:
|
|
"""The [tool.hermes.extras-platforms] table from pyproject.toml:
|
|
extra -> PEP 508 marker string. Cached per process."""
|
|
global _PLATFORM_GATES
|
|
if _PLATFORM_GATES is not None:
|
|
return _PLATFORM_GATES
|
|
import tomllib
|
|
|
|
from pm.paths import repo_root
|
|
|
|
gates: dict[str, str] = {}
|
|
try:
|
|
with (repo_root() / "pyproject.toml").open("rb") as f:
|
|
data = tomllib.load(f)
|
|
table = data.get("tool", {}).get("hermes", {}).get("extras-platforms", {})
|
|
if isinstance(table, dict):
|
|
gates = {str(k): str(v) for k, v in table.items()}
|
|
except (OSError, ValueError):
|
|
pass
|
|
_PLATFORM_GATES = gates
|
|
return gates
|
|
|
|
|
|
_PLATFORM_GATES: dict[str, str] | None = None
|
|
|
|
|
|
def extra_supported(extra: str, *, environment: dict[str, str] | None = None,
|
|
importable: Callable[[str], bool] | None = None) -> bool:
|
|
"""Is this extra installable on THIS platform? True when the extra
|
|
carries no gate, or its marker matches the running platform. An
|
|
extra that IS present on this machine (anchors importable) is always
|
|
supported — an installed override beats the table (dev machines,
|
|
hand-synced venvs)."""
|
|
if all((importable or _importable)(a) for a in _anchors(extra)):
|
|
return True
|
|
marker = _platform_gates().get(extra)
|
|
if marker is None:
|
|
return True
|
|
import os
|
|
import platform
|
|
import sys
|
|
|
|
if environment is None:
|
|
environment = {
|
|
"sys_platform": sys.platform,
|
|
"platform_system": platform.system(),
|
|
"platform_machine": platform.machine(),
|
|
"os_name": os.name,
|
|
}
|
|
try:
|
|
from packaging.markers import Marker
|
|
except ImportError:
|
|
# The historical update takeover runs PM in the OLD venv's
|
|
# interpreter, which need not ship `packaging`. PM's runtime does.
|
|
return _evaluate_in_runtime(marker, environment)
|
|
try:
|
|
return bool(Marker(marker).evaluate(environment=environment))
|
|
except Exception:
|
|
# A malformed marker must never brick availability — treat as
|
|
# ungated and let the resolver be the authority.
|
|
return True
|
|
|
|
|
|
def _evaluate_in_runtime(marker: str, environment: dict[str, str]) -> bool:
|
|
import json
|
|
import subprocess
|
|
|
|
from pm.runtime import runtime_command, runtime_environment
|
|
|
|
try:
|
|
command = runtime_command(Path(__file__).with_name("_marker_eval.py"),
|
|
[marker, json.dumps(environment)])
|
|
result = subprocess.run(command, env=runtime_environment(), capture_output=True,
|
|
text=True, timeout=60, check=False)
|
|
except Exception:
|
|
return True # same fallback as a malformed marker: the resolver decides
|
|
if result.returncode != 0:
|
|
return True
|
|
return result.stdout.strip() == "1"
|
|
|
|
|
|
def install_hint(extra: str) -> str:
|
|
"""The one command users are told to run for a missing extra."""
|
|
return f"hermes pm install --extra {extra}"
|
|
|
|
|
|
def ensure_import(extra: str) -> None:
|
|
"""Make an extra available: no-op when the anchor imports, otherwise
|
|
sync the venv with the extra enabled. Raises InstallError on failure
|
|
— including when a platform gate excludes this machine."""
|
|
if available(extra):
|
|
return
|
|
if not extra_supported(extra):
|
|
from pm.package import InstallError
|
|
|
|
marker = _platform_gates().get(extra, "")
|
|
raise InstallError(
|
|
"venv",
|
|
f"extra {extra!r} is not supported on this platform "
|
|
f"(gate: {marker!r}); the adapter degrades without it",
|
|
)
|
|
import sys
|
|
from pm.package import InstallError
|
|
|
|
# prompt_toolkit already owns stdin during a CLI turn; never read from it.
|
|
app_running = False
|
|
if "prompt_toolkit.application.current" in sys.modules:
|
|
from prompt_toolkit.application.current import get_app_or_none
|
|
|
|
app_running = bool(getattr(get_app_or_none(), "is_running", False))
|
|
if not app_running and sys.stdin.isatty() and sys.stdout.isatty():
|
|
try:
|
|
answer = input(f"\nThis needs Hermes' optional {extra!r} feature, which isn't installed yet.\n"
|
|
"Install it now? [Y/n] ").strip().lower()
|
|
except (EOFError, KeyboardInterrupt):
|
|
answer = "n"
|
|
if answer and answer not in {"y", "yes"}:
|
|
raise InstallError("venv", f"installation of extra {extra!r} declined")
|
|
from pm.client import sync_venv
|
|
|
|
sync_venv([extra])
|
|
# The sync published a new generation. Swap this process onto it when nothing
|
|
# already imported would change underneath it (adopt_selected); otherwise only
|
|
# a restart can load it.
|
|
from pm.environments import selected_venv, site_packages
|
|
from pm.environments_adopt import adopt_selected, note_restart_pending, restart_needed
|
|
from pm.paths import repo_root, runtime_facts_path
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
if runtime_facts_path().is_file():
|
|
root = repo_root()
|
|
adopt_selected(root)
|
|
selected = site_packages(selected_venv(root)).resolve()
|
|
if selected not in {Path(entry).resolve() for entry in sys.path}:
|
|
reason = restart_needed(root) or "this process does not run from the install's dependency environment"
|
|
note_restart_pending(f"{extra} installed ({reason})")
|
|
raise InstallError("venv", f"{extra} installed; restart Hermes to activate it ({reason})")
|
|
|
|
|
|
def ensure_and_bind(extra, importer, target_globals) -> bool:
|
|
"""ensure_import + rebind module-level names after a mid-process install.
|
|
importer returns {name: value}; bound into target_globals on success."""
|
|
try:
|
|
ensure_import(extra)
|
|
except Exception as exc:
|
|
import logging
|
|
|
|
logging.getLogger(__name__).warning("extra %r unavailable: %s", extra, exc)
|
|
return False
|
|
try:
|
|
bindings = importer()
|
|
except ImportError as exc:
|
|
import logging
|
|
|
|
logging.getLogger(__name__).warning(
|
|
"import after installing %r failed: %s", extra, exc
|
|
)
|
|
return False
|
|
target_globals.update(bindings)
|
|
return True
|
|
|
|
|
|
def missing(extra: str) -> tuple[str, ...]:
|
|
return tuple(a for a in _anchors(extra) if not _importable(a))
|
|
|
|
|
|
def _installed_in(site_packages: Path, anchor: str) -> bool:
|
|
"""Is ``anchor`` installed under a site-packages we must not import from?"""
|
|
*parents, leaf = anchor.split(".")
|
|
parent = site_packages.joinpath(*parents)
|
|
if (parent / leaf).is_dir():
|
|
return True
|
|
# A top-level module can be a plain file or a compiled extension.
|
|
return any(entry.name.rsplit(".", 1)[-1] in {"py", "so", "pyd"}
|
|
for entry in parent.glob(f"{leaf}.*"))
|
|
|
|
|
|
DECLINED_EXTRAS_FILENAME = "declined-extras.json"
|
|
|
|
|
|
def declined_extras_path(project_root: Path | None = None) -> Path:
|
|
from pm.environments import install_state_dir
|
|
from pm.paths import repo_root
|
|
|
|
return install_state_dir(repo_root() if project_root is None else Path(project_root)) / DECLINED_EXTRAS_FILENAME
|
|
|
|
|
|
def declined_extras(project_root: Path | None = None) -> frozenset[str]:
|
|
"""Extras the user dropped with ``hermes pm install --without NAME``.
|
|
|
|
Recorded beside the dependency facts, so a carried selection (legacy
|
|
migration, recorded extras, configured-feature auto-installs) never
|
|
re-adds one; only an explicit request (``--extra NAME``) opts back in.
|
|
"""
|
|
import json
|
|
|
|
path = declined_extras_path(project_root)
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except FileNotFoundError:
|
|
return frozenset()
|
|
except (OSError, ValueError) as exc:
|
|
# Reading a damaged record as "nothing declined" would re-add what the user removed.
|
|
from pm.package import InstallError
|
|
|
|
raise InstallError("venv", f"cannot read {path}: {exc}", "fix or delete the file, then retry") from exc
|
|
names = data.get("declined") if isinstance(data, dict) else None
|
|
return frozenset(_canonical(name) for name in names or () if isinstance(name, str))
|
|
|
|
|
|
def record_declined_extras(*, add=(), remove=(), reasons: dict[str, str] | None = None,
|
|
project_root: Path | None = None) -> frozenset[str]:
|
|
"""Update the declined extras; returns the new set. Writes only on change.
|
|
|
|
*reasons* marks extras an update left out on its own (``pm.carried_extras``) with why,
|
|
so a later refusal names the cause instead of a ``--without`` the user never ran.
|
|
"""
|
|
import json
|
|
|
|
from pm.filesystem import durable_write_bytes
|
|
|
|
current = declined_extras(project_root)
|
|
updated = (current | {_canonical(name) for name in add}) - {_canonical(name) for name in remove}
|
|
known = declined_reasons(project_root)
|
|
noted = {name: text for name, text in {**known, **{_canonical(name): " ".join(text.split())[:300]
|
|
for name, text in (reasons or {}).items()}}.items()
|
|
if name in updated}
|
|
if updated != current or noted != known:
|
|
record: dict = {"schema": 1, "declined": sorted(updated)}
|
|
if noted:
|
|
record["left_out_by_update"] = dict(sorted(noted.items()))
|
|
durable_write_bytes(declined_extras_path(project_root), (json.dumps(record, indent=2) + "\n").encode("utf-8"))
|
|
return frozenset(updated)
|
|
|
|
|
|
def declined_reasons(project_root: Path | None = None) -> dict[str, str]:
|
|
"""Why an update left each automatically declined extra out (absent for a user's ``--without``)."""
|
|
import json
|
|
|
|
try:
|
|
data = json.loads(declined_extras_path(project_root).read_text(encoding="utf-8-sig"))
|
|
except (OSError, ValueError):
|
|
return {} # declined_extras() is the reader that refuses a damaged record
|
|
notes = data.get("left_out_by_update") if isinstance(data, dict) else None
|
|
return {name: text for name, text in (notes or {}).items() if isinstance(name, str) and isinstance(text, str)}
|
|
|
|
|
|
def _canonical(name: str) -> str:
|
|
"""The PEP 685 name uv matches an extra by (``foo_bar`` is ``foo-bar``)."""
|
|
import re
|
|
|
|
return re.sub(r"[-_.]+", "-", name).lower()
|
|
|
|
|
|
def droppable_extras(project_root: Path) -> list[str]:
|
|
"""Declared extras ``--without`` may drop: not ``all`` and nothing ``all`` pulls in.
|
|
|
|
The default sync selects ``[all]``, so dropping one of its members would be a no-op.
|
|
"""
|
|
import re
|
|
import tomllib
|
|
|
|
with (Path(project_root) / "pyproject.toml").open("rb") as f:
|
|
project = tomllib.load(f).get("project", {})
|
|
table = project.get("optional-dependencies", {})
|
|
self_ref = re.compile(rf"^{re.escape(project.get('name', ''))}\[([^\]]+)\]", re.IGNORECASE)
|
|
kept, pending = {"all"}, ["all"]
|
|
while pending:
|
|
for requirement in table.get(pending.pop(), ()):
|
|
match = self_ref.match(requirement.replace(" ", ""))
|
|
for member in (match.group(1).split(",") if match else ()):
|
|
if member not in kept:
|
|
kept.add(member)
|
|
pending.append(member)
|
|
return sorted(extra for extra in table if extra not in kept)
|
|
|
|
|
|
def legacy_selection(project_root: Path) -> list[str]:
|
|
"""The extras PM's first generation selects when it replaces a main-era venv.
|
|
|
|
Main-era installers selected ``[all]`` and then lazily installed opt-in
|
|
extras (FAL, messaging SDKs, ...) into the checkout's own venv, with no
|
|
ledger. Selecting only ``[all]`` drops those, and the first launch after
|
|
the migration asks to reinstall a feature that already worked. The old
|
|
venv's site-packages is read, never imported: the migrating process may
|
|
not run from it.
|
|
"""
|
|
root = Path(project_root)
|
|
trees = [tree for venv in (root / "venv", root / ".venv")
|
|
for tree in (*venv.glob("lib/python*/site-packages"), venv / "Lib" / "site-packages")
|
|
if tree.is_dir()]
|
|
# The old venv is never deleted, so this scan repeats for any data root without facts.
|
|
refused = declined_extras(root)
|
|
carried = sorted(
|
|
# An umbrella extra shares its anchor with one member; carrying it
|
|
# would install every sibling the user never chose.
|
|
extra for extra in ANCHORS if extra not in {"messaging", "voice", "wake"} and extra not in refused
|
|
# PM refuses a gated extra outside its platform even if a hand-synced venv carried it.
|
|
# Installed first: judging a gate may cost a PM-runtime subprocess.
|
|
if any(all(_installed_in(tree, anchor) for anchor in _anchors(extra)) for tree in trees)
|
|
and extra_supported(extra, importable=lambda _anchor: False)
|
|
)
|
|
return ["all", *carried]
|