hermes-agent/scripts/ci/setup_toolchain.py

289 lines
12 KiB
Python
Raw Permalink Normal View History

"""GitHub Actions file commands around the real, stdlib-only PM bootstrap."""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import platform
import re
import sys
# The runner invokes this file before the checkout has been installed.
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from pm.lock import Lockfile
from pm.paths import lockfile_path
from pm.registry import walk
from pm.store import current_target
_ROOTS = {"python": ["uv"], "node": ["npm"], "all": ["npm", "uv"]}
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
def parse_package_list(value: str) -> list[str]:
"""Comma-separated extra PM tools (e.g. `ffmpeg`) beyond the toolchain roots."""
if value == "":
return []
for name in value.split(","):
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name.strip()):
raise argparse.ArgumentTypeError(f"invalid pm tool name: {name!r}")
return sorted({name.strip() for name in value.split(",")})
def packages(toolchain: str, extra: list[str] | None = None) -> list[str]:
# walk() resolves the deps-first closure, so an extra tool's dependencies
# (ffmpeg -> none) come along without listing them.
return sorted(package.name for package in walk(_ROOTS[toolchain] + (extra or [])))
def file_commands(destination: str, values: dict) -> None:
"""All exports are single-line values, including JSON-encoded arrays."""
lines = []
for key, value in values.items():
text = str(value)
if any(character in key + text for character in "\r\n\0"):
raise ValueError(f"invalid GitHub file command: {key!r}")
lines.append(f"{key}={text}\n")
with open(os.environ[destination], "a", encoding="utf-8") as stream:
stream.writelines(lines)
def parse_extras(value: str) -> list[str] | None:
if value == "":
return None
message = "extras must be a JSON array of extra names"
try:
extras = json.loads(value)
except ValueError as exc:
raise argparse.ArgumentTypeError(message) from exc
if not isinstance(extras, list) or any(
not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name)
for name in extras
):
raise argparse.ArgumentTypeError(message)
return sorted(set(extras))
def prepare(args) -> None:
home = args.home.resolve()
lock = Lockfile(lockfile_path())
target = current_target()
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
names = packages(args.toolchain, args.packages)
values = {
"packages": json.dumps(names), "target": target, "arch": target.split("-")[1],
"store": str(home / "tools"),
}
for name in names:
version = lock.version(name)
if not version or not lock.artifacts(name, target):
raise ValueError(f"{name} has no pinned artifact for {target}")
values[f"{name}-version"] = version.partition("+")[0] if name == "python" else version
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
# Cache-key fragment for the tools cache: empty when no extras, so every
# existing key stays byte-identical.
values["extra-packages"] = "".join(f"-{name}" for name in (args.packages or []))
# The OS image belongs in the uv cache identity: built wheels can link
# against its system libraries. Unlike npm's cache, these are not just JS.
values["os-version"] = platform.platform()
values["bootstrap-python"] = sys.executable
file_commands("GITHUB_OUTPUT", values)
file_commands("GITHUB_ENV", {
"HERMES_HOME": home,
"HERMES_RUNTIME_DIR": home / "tools",
"PYTHONUTF8": "1",
})
def add_path(directories: list[str]) -> None:
# The runner prepends each line, so write PM's dependent-first PATH in
# reverse. npm must shadow the different npm bundled inside Node.
with open(os.environ["GITHUB_PATH"], "a", encoding="utf-8") as stream:
for directory in reversed(list(dict.fromkeys(directories))):
if any(character in directory for character in "\r\n\0"):
raise ValueError("invalid PATH directory")
stream.write(directory + "\n")
def python3_alias(python: Path) -> None:
if os.name == "nt":
import shutil
alias = python.with_name("python3.exe")
if not alias.exists():
shutil.copy2(python, alias)
def archive_inputs(args) -> None:
from pm.paths import repo_root
from pm.store import Store
from scripts.ci.archive_inputs import Archive, pinned_inputs, stage_inputs
from scripts.releases import r2
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
pins = pinned_inputs(repo_root(), target=current_target(), packages=set(packages(args.toolchain, args.packages)))
stage_inputs(pins, archive=Archive(*r2.credentials()), store=Store(args.home.resolve() / "tools"))
# IO_REPARSE_TAG_LX_SYMLINK: a WSL-style symlink, which Python cannot open.
_LX_SYMLINK = 0xA000001D
def restore_cygwin_symlinks(root: Path) -> int:
"""Rewrite the WSL symlinks a cache restore left in the PM store as Cygwin ones.
Git for Windows ships symlinks such as etc/mtab as Cygwin system files (a
``!<symlink>`` file with the SYSTEM attribute), and PM's digest covers those
bytes. @actions/cache restores the store with Git's MSYS tar under
MSYS=winsymlinks:nativestrict, which writes a symlink to a non-Windows target
as a WSL reparse point instead. Python cannot open that, so the digest check
failed and every Windows job re-extracted Git (~21s on arm64). Recreate each
one with Git's own ln under winsymlinks:sys. PM still verifies the entry, so a
link that comes back different only costs the re-extraction it used to.
"""
if os.name != "nt" or not root.is_dir():
return 0
import shutil
import subprocess
readlink, ln = shutil.which("readlink"), shutil.which("ln")
if not (readlink and ln):
return 0
env = dict(os.environ, MSYS="winsymlinks:sys")
repaired = 0
for dirpath, _dirnames, filenames in os.walk(root):
for name in filenames:
path = os.path.join(dirpath, name)
try:
if os.lstat(path).st_reparse_tag != _LX_SYMLINK:
continue
target = subprocess.run([readlink, path], env=env, capture_output=True,
text=True, check=True, timeout=30).stdout.rstrip("\n")
os.remove(path)
subprocess.run([ln, "-s", target, path], env=env, check=True, timeout=30)
repaired += 1
except (OSError, subprocess.SubprocessError):
continue
return repaired
def install(args) -> None:
import subprocess
from pm import build_requirements_environment
from pm.cli import _live_progress
from pm.install import ensure, env_for
from pm.lock import Facts
from pm.package import compose_env
from pm.packages import uv_cache_dir
from pm.paths import facts_path, store_root
from pm.registry import get_package
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
names = packages(args.toolchain, args.packages)
repaired = restore_cygwin_symlinks(store_root())
if repaired:
print(f"Restored {repaired} Cygwin symlink(s) in the cached PM store")
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
for name in _ROOTS[args.toolchain] + args.packages:
ensure(name, explicit=True, progress=_live_progress(name))
facts = Facts(facts_path())
target = current_target()
public_names = [name for name in names if name != "uv"]
binaries = {
name: get_package(name).binary(store_root() / facts.get(name)["entry"], target)
for name in public_names
}
environment = env_for(*public_names, base_env={})
path = environment["PATH"].split(os.pathsep)
environment = compose_env([environment])
exported = {}
outputs = {f"{name}-path": str(binaries[name]) for name in public_names}
if "python" in names:
# uv is PM-internal (never on a user's PATH), but the test suites this
# toolchain serves drive real uv through shutil.which("uv").
uv = get_package("uv").binary(store_root() / facts.get("uv")["entry"], target)
path.append(str(uv.parent))
outputs["uv-path"] = str(uv)
# Keep third-party CI tooling out of the verified interpreter store.
# PM prepares the empty command environment through its normal builder.
commands = args.home.resolve() / "python" / facts.get("python")["entry"]
if not (commands / "pyvenv.cfg").is_file():
build_requirements_environment([], out=commands, python=binaries["python"],
env=environment, explicit=True)
python = commands / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
python3_alias(python)
path.insert(0, str(python.parent))
outputs["python-path"] = str(python)
exported.update({
"HERMES_PYTHON": python,
})
outputs["uv-cache-path"] = str(uv_cache_dir())
if "npm" in names:
cache = subprocess.check_output(
[str(binaries["npm"]), "config", "get", "cache"],
env=environment, text=True, encoding="utf-8", timeout=60,
).strip()
outputs["npm-cache-path"] = cache
file_commands("GITHUB_ENV", exported)
file_commands("GITHUB_OUTPUT", outputs)
add_path(path)
print("PM toolchain ready: " + ", ".join(f"{name} {facts.get(name)['version']}" for name in names))
def dependencies(args) -> None:
if args.extras is None and not args.test_environment:
return
import tomllib
from pm.environments import selected_venv
from pm import check_project_lock, sync_venv
from pm.plugin_inputs import Members
from pm.paths import repo_root
project = repo_root()
metadata = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8-sig"))
extras = args.extras or []
unknown = set(extras) - metadata["project"].get("optional-dependencies", {}).keys()
if unknown:
raise ValueError(f"unknown project extras: {sorted(unknown)}")
# Frozen sync must not turn a stale project lock into a green job.
check_project_lock(project, explicit=True)
if args.test_environment:
from pm.testenv import ensure_testenv
# CI and activation select the same side environment, never PM facts.
python = ensure_testenv(project, extras)
venv = python.parent.parent
else:
sync_venv(extras, explicit=True, plugins=Members([]))
venv = selected_venv(project)
bindir = venv / ("Scripts" if os.name == "nt" else "bin")
python = bindir / ("python.exe" if os.name == "nt" else "python")
python3_alias(python)
file_commands("GITHUB_ENV", {
"HERMES_PYTHON": python,
"VIRTUAL_ENV": venv,
"PYTHONPATH": project,
})
file_commands("GITHUB_OUTPUT", {"python-path": python, "venv": venv})
add_path([str(bindir)])
print(f"PM dependencies ready: {args.extras} in {venv}")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
phases = {"prepare": prepare, "archive-inputs": archive_inputs, "install": install, "dependencies": dependencies}
parser.add_argument("phase", choices=list(phases))
parser.add_argument("--toolchain", choices=["python", "node", "all"], default="python")
parser.add_argument("--home", type=Path, required=True)
parser.add_argument("--extras", type=parse_extras, default="")
parser.add_argument("--test-environment", action="store_true")
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs The e2e-screen-record action installed ffmpeg through three different OS package managers (apt, brew, winget). winget is the flaky leg on windows-11-arm and serves an x64-gyan build that runs emulated on ARM; choco's community package wraps the same gyan x64 zip, so a choco fallback would not fix either problem. PM already ships a locked, sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE winarm64 build), so the recording action now verifies ffmpeg on PATH instead of installing it, and the pinned binary rides the same tools-cache as node/python. - setup-pm learns a `packages` input (extra PM tools beyond the toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's prepare/install/archive-inputs phases; the tools-cache key gains an extra-packages fragment so existing keys stay byte-identical. - The six chat-driver jobs pass `packages: ffmpeg` to setup-pm. - e2e-screen-record drops the apt/brew/winget install steps, the ffmpeg actions/cache steps and the save-cache input; Xvfb (headless linux) and the macOS replayd-approval hack stay. - The "Install locked chat driver dependencies" step installs only the tests-js workspace with --omit=dev instead of the whole apps/desktop tree: the drivers need @playwright/test, zod (previously a phantom hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure packages in ~2s vs ~1800 including electron-builder and native builds; the tree-shaken node_modules runs the real driver modules (verified by importing desktop-chat-smoke.ts and update-window-chat end to end). - tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
parser.add_argument("--packages", type=parse_package_list, default=[], help="extra PM tools beyond the toolchain roots (e.g. ffmpeg)")
args = parser.parse_args()
if args.toolchain == "node" and (args.extras is not None or args.test_environment):
parser.error("Python dependencies require the python or all toolchain")
os.environ["HERMES_HOME"] = str(args.home.resolve())
os.environ["HERMES_RUNTIME_DIR"] = str(args.home.resolve() / "tools")
phases[args.phase](args)
if __name__ == "__main__":
main()