2026-09-08 12:45:15 -04:00
|
|
|
"""GitHub Actions file commands around the real, stdlib-only PM bootstrap."""
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
import platform
|
|
|
|
|
import re
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
# The runner invokes this file before the checkout has been installed.
|
|
|
|
|
if __package__ in (None, ""):
|
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
|
|
|
|
|
|
from pm.lock import Lockfile
|
|
|
|
|
from pm.paths import lockfile_path
|
|
|
|
|
from pm.registry import walk
|
|
|
|
|
from pm.store import current_target
|
|
|
|
|
|
2026-09-12 18:34:01 -04:00
|
|
|
_ROOTS = {"python": ["uv"], "node": ["npm"], "all": ["npm", "uv"]}
|
|
|
|
|
|
2026-09-08 12:45:15 -04:00
|
|
|
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
def parse_package_list(value: str) -> list[str]:
|
|
|
|
|
"""Comma-separated extra PM tools (e.g. `ffmpeg`) beyond the toolchain roots."""
|
|
|
|
|
if value == "":
|
|
|
|
|
return []
|
|
|
|
|
for name in value.split(","):
|
|
|
|
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name.strip()):
|
|
|
|
|
raise argparse.ArgumentTypeError(f"invalid pm tool name: {name!r}")
|
|
|
|
|
return sorted({name.strip() for name in value.split(",")})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def packages(toolchain: str, extra: list[str] | None = None) -> list[str]:
|
|
|
|
|
# walk() resolves the deps-first closure, so an extra tool's dependencies
|
|
|
|
|
# (ffmpeg -> none) come along without listing them.
|
|
|
|
|
return sorted(package.name for package in walk(_ROOTS[toolchain] + (extra or [])))
|
2026-09-08 12:45:15 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def file_commands(destination: str, values: dict) -> None:
|
|
|
|
|
"""All exports are single-line values, including JSON-encoded arrays."""
|
|
|
|
|
lines = []
|
|
|
|
|
for key, value in values.items():
|
|
|
|
|
text = str(value)
|
|
|
|
|
if any(character in key + text for character in "\r\n\0"):
|
|
|
|
|
raise ValueError(f"invalid GitHub file command: {key!r}")
|
|
|
|
|
lines.append(f"{key}={text}\n")
|
|
|
|
|
with open(os.environ[destination], "a", encoding="utf-8") as stream:
|
|
|
|
|
stream.writelines(lines)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_extras(value: str) -> list[str] | None:
|
|
|
|
|
if value == "":
|
|
|
|
|
return None
|
|
|
|
|
message = "extras must be a JSON array of extra names"
|
|
|
|
|
try:
|
|
|
|
|
extras = json.loads(value)
|
|
|
|
|
except ValueError as exc:
|
|
|
|
|
raise argparse.ArgumentTypeError(message) from exc
|
|
|
|
|
if not isinstance(extras, list) or any(
|
|
|
|
|
not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name)
|
|
|
|
|
for name in extras
|
|
|
|
|
):
|
|
|
|
|
raise argparse.ArgumentTypeError(message)
|
|
|
|
|
return sorted(set(extras))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def prepare(args) -> None:
|
|
|
|
|
home = args.home.resolve()
|
|
|
|
|
lock = Lockfile(lockfile_path())
|
|
|
|
|
target = current_target()
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
names = packages(args.toolchain, args.packages)
|
2026-09-08 12:45:15 -04:00
|
|
|
values = {
|
|
|
|
|
"packages": json.dumps(names), "target": target, "arch": target.split("-")[1],
|
|
|
|
|
"store": str(home / "tools"),
|
|
|
|
|
}
|
|
|
|
|
for name in names:
|
|
|
|
|
version = lock.version(name)
|
|
|
|
|
if not version or not lock.artifacts(name, target):
|
|
|
|
|
raise ValueError(f"{name} has no pinned artifact for {target}")
|
|
|
|
|
values[f"{name}-version"] = version.partition("+")[0] if name == "python" else version
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
# Cache-key fragment for the tools cache: empty when no extras, so every
|
|
|
|
|
# existing key stays byte-identical.
|
|
|
|
|
values["extra-packages"] = "".join(f"-{name}" for name in (args.packages or []))
|
2026-09-08 12:45:15 -04:00
|
|
|
# The OS image belongs in the uv cache identity: built wheels can link
|
|
|
|
|
# against its system libraries. Unlike npm's cache, these are not just JS.
|
|
|
|
|
values["os-version"] = platform.platform()
|
|
|
|
|
values["bootstrap-python"] = sys.executable
|
|
|
|
|
file_commands("GITHUB_OUTPUT", values)
|
|
|
|
|
file_commands("GITHUB_ENV", {
|
|
|
|
|
"HERMES_HOME": home,
|
|
|
|
|
"HERMES_RUNTIME_DIR": home / "tools",
|
|
|
|
|
"PYTHONUTF8": "1",
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def add_path(directories: list[str]) -> None:
|
|
|
|
|
# The runner prepends each line, so write PM's dependent-first PATH in
|
|
|
|
|
# reverse. npm must shadow the different npm bundled inside Node.
|
|
|
|
|
with open(os.environ["GITHUB_PATH"], "a", encoding="utf-8") as stream:
|
|
|
|
|
for directory in reversed(list(dict.fromkeys(directories))):
|
|
|
|
|
if any(character in directory for character in "\r\n\0"):
|
|
|
|
|
raise ValueError("invalid PATH directory")
|
|
|
|
|
stream.write(directory + "\n")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def python3_alias(python: Path) -> None:
|
|
|
|
|
if os.name == "nt":
|
|
|
|
|
import shutil
|
|
|
|
|
|
|
|
|
|
alias = python.with_name("python3.exe")
|
|
|
|
|
if not alias.exists():
|
|
|
|
|
shutil.copy2(python, alias)
|
|
|
|
|
|
|
|
|
|
|
fix(pm): preserve pinned binary inputs in R2
Termux removes old package files, so a pinned URL and hash do not keep
build inputs available. Preserve the exact bytes without changing pins.
Archive every PM HTTP artifact and the Termux runtime inputs by SHA256.
CI reads R2 first. Only a missing object permits an upstream download,
hash verification, immutable upload, and verified readback. Seed the
actual toolchain and payload stores before their consumers run.
Use the public archive as a pinned fallback in PM, bootstrap installers,
and Nix fetchers. Keep network retries bounded and report attempted URLs.
Keep publication credentials in protected CI jobs, not installed clients.
Verification:
- 283 targeted tests passed; five POSIX tests skipped on Windows.
- All 87 preserved Termux packages passed local archive miss/hit checks.
- Native ARM64 ripgrep installed through the mirror and ran successfully.
- Wheel import, workflow lint, Python lint, shell syntax, and pins passed.
Live R2 publication, POSIX tests, and Nix builds remain for native CI.
The real-byte archive checks used loopback HTTP, not the live bucket.
2026-09-10 18:17:08 -04:00
|
|
|
def archive_inputs(args) -> None:
|
|
|
|
|
from pm.paths import repo_root
|
|
|
|
|
from pm.store import Store
|
|
|
|
|
from scripts.ci.archive_inputs import Archive, pinned_inputs, stage_inputs
|
|
|
|
|
from scripts.releases import r2
|
|
|
|
|
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
pins = pinned_inputs(repo_root(), target=current_target(), packages=set(packages(args.toolchain, args.packages)))
|
fix(pm): preserve pinned binary inputs in R2
Termux removes old package files, so a pinned URL and hash do not keep
build inputs available. Preserve the exact bytes without changing pins.
Archive every PM HTTP artifact and the Termux runtime inputs by SHA256.
CI reads R2 first. Only a missing object permits an upstream download,
hash verification, immutable upload, and verified readback. Seed the
actual toolchain and payload stores before their consumers run.
Use the public archive as a pinned fallback in PM, bootstrap installers,
and Nix fetchers. Keep network retries bounded and report attempted URLs.
Keep publication credentials in protected CI jobs, not installed clients.
Verification:
- 283 targeted tests passed; five POSIX tests skipped on Windows.
- All 87 preserved Termux packages passed local archive miss/hit checks.
- Native ARM64 ripgrep installed through the mirror and ran successfully.
- Wheel import, workflow lint, Python lint, shell syntax, and pins passed.
Live R2 publication, POSIX tests, and Nix builds remain for native CI.
The real-byte archive checks used loopback HTTP, not the live bucket.
2026-09-10 18:17:08 -04:00
|
|
|
stage_inputs(pins, archive=Archive(*r2.credentials()), store=Store(args.home.resolve() / "tools"))
|
|
|
|
|
|
|
|
|
|
|
2026-11-03 13:30:41 +05:30
|
|
|
# IO_REPARSE_TAG_LX_SYMLINK: a WSL-style symlink, which Python cannot open.
|
|
|
|
|
_LX_SYMLINK = 0xA000001D
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def restore_cygwin_symlinks(root: Path) -> int:
|
|
|
|
|
"""Rewrite the WSL symlinks a cache restore left in the PM store as Cygwin ones.
|
|
|
|
|
|
|
|
|
|
Git for Windows ships symlinks such as etc/mtab as Cygwin system files (a
|
|
|
|
|
``!<symlink>`` file with the SYSTEM attribute), and PM's digest covers those
|
|
|
|
|
bytes. @actions/cache restores the store with Git's MSYS tar under
|
|
|
|
|
MSYS=winsymlinks:nativestrict, which writes a symlink to a non-Windows target
|
|
|
|
|
as a WSL reparse point instead. Python cannot open that, so the digest check
|
|
|
|
|
failed and every Windows job re-extracted Git (~21s on arm64). Recreate each
|
|
|
|
|
one with Git's own ln under winsymlinks:sys. PM still verifies the entry, so a
|
|
|
|
|
link that comes back different only costs the re-extraction it used to.
|
|
|
|
|
"""
|
|
|
|
|
if os.name != "nt" or not root.is_dir():
|
|
|
|
|
return 0
|
|
|
|
|
import shutil
|
|
|
|
|
import subprocess
|
|
|
|
|
|
|
|
|
|
readlink, ln = shutil.which("readlink"), shutil.which("ln")
|
|
|
|
|
if not (readlink and ln):
|
|
|
|
|
return 0
|
|
|
|
|
env = dict(os.environ, MSYS="winsymlinks:sys")
|
|
|
|
|
repaired = 0
|
|
|
|
|
for dirpath, _dirnames, filenames in os.walk(root):
|
|
|
|
|
for name in filenames:
|
|
|
|
|
path = os.path.join(dirpath, name)
|
|
|
|
|
try:
|
|
|
|
|
if os.lstat(path).st_reparse_tag != _LX_SYMLINK:
|
|
|
|
|
continue
|
|
|
|
|
target = subprocess.run([readlink, path], env=env, capture_output=True,
|
|
|
|
|
text=True, check=True, timeout=30).stdout.rstrip("\n")
|
|
|
|
|
os.remove(path)
|
|
|
|
|
subprocess.run([ln, "-s", target, path], env=env, check=True, timeout=30)
|
|
|
|
|
repaired += 1
|
|
|
|
|
except (OSError, subprocess.SubprocessError):
|
|
|
|
|
continue
|
|
|
|
|
return repaired
|
|
|
|
|
|
|
|
|
|
|
2026-09-08 12:45:15 -04:00
|
|
|
def install(args) -> None:
|
|
|
|
|
import subprocess
|
|
|
|
|
|
2026-09-11 18:14:43 -04:00
|
|
|
from pm import build_requirements_environment
|
2026-09-08 12:45:15 -04:00
|
|
|
from pm.cli import _live_progress
|
2026-09-18 23:27:05 -04:00
|
|
|
from pm.install import ensure, env_for
|
2026-09-08 12:45:15 -04:00
|
|
|
from pm.lock import Facts
|
2026-09-12 18:34:01 -04:00
|
|
|
from pm.package import compose_env
|
2026-09-08 12:45:15 -04:00
|
|
|
from pm.packages import uv_cache_dir
|
|
|
|
|
from pm.paths import facts_path, store_root
|
|
|
|
|
from pm.registry import get_package
|
|
|
|
|
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
names = packages(args.toolchain, args.packages)
|
2026-11-03 13:30:41 +05:30
|
|
|
repaired = restore_cygwin_symlinks(store_root())
|
|
|
|
|
if repaired:
|
|
|
|
|
print(f"Restored {repaired} Cygwin symlink(s) in the cached PM store")
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
for name in _ROOTS[args.toolchain] + args.packages:
|
2026-09-08 12:45:15 -04:00
|
|
|
ensure(name, explicit=True, progress=_live_progress(name))
|
|
|
|
|
facts = Facts(facts_path())
|
|
|
|
|
target = current_target()
|
2026-09-11 18:14:43 -04:00
|
|
|
public_names = [name for name in names if name != "uv"]
|
2026-09-08 12:45:15 -04:00
|
|
|
binaries = {
|
|
|
|
|
name: get_package(name).binary(store_root() / facts.get(name)["entry"], target)
|
2026-09-11 18:14:43 -04:00
|
|
|
for name in public_names
|
2026-09-08 12:45:15 -04:00
|
|
|
}
|
2026-09-12 18:34:01 -04:00
|
|
|
environment = env_for(*public_names, base_env={})
|
|
|
|
|
path = environment["PATH"].split(os.pathsep)
|
|
|
|
|
environment = compose_env([environment])
|
2026-09-08 12:45:15 -04:00
|
|
|
exported = {}
|
2026-09-11 18:14:43 -04:00
|
|
|
outputs = {f"{name}-path": str(binaries[name]) for name in public_names}
|
2026-09-08 12:45:15 -04:00
|
|
|
if "python" in names:
|
2026-09-19 01:52:39 -04:00
|
|
|
# uv is PM-internal (never on a user's PATH), but the test suites this
|
|
|
|
|
# toolchain serves drive real uv through shutil.which("uv").
|
|
|
|
|
uv = get_package("uv").binary(store_root() / facts.get("uv")["entry"], target)
|
|
|
|
|
path.append(str(uv.parent))
|
|
|
|
|
outputs["uv-path"] = str(uv)
|
2026-09-11 18:14:43 -04:00
|
|
|
# Keep third-party CI tooling out of the verified interpreter store.
|
|
|
|
|
# PM prepares the empty command environment through its normal builder.
|
2026-09-08 12:45:15 -04:00
|
|
|
commands = args.home.resolve() / "python" / facts.get("python")["entry"]
|
|
|
|
|
if not (commands / "pyvenv.cfg").is_file():
|
2026-09-11 18:14:43 -04:00
|
|
|
build_requirements_environment([], out=commands, python=binaries["python"],
|
|
|
|
|
env=environment, explicit=True)
|
2026-09-08 12:45:15 -04:00
|
|
|
python = commands / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
|
|
|
python3_alias(python)
|
|
|
|
|
path.insert(0, str(python.parent))
|
|
|
|
|
outputs["python-path"] = str(python)
|
|
|
|
|
exported.update({
|
|
|
|
|
"HERMES_PYTHON": python,
|
|
|
|
|
})
|
|
|
|
|
outputs["uv-cache-path"] = str(uv_cache_dir())
|
|
|
|
|
if "npm" in names:
|
|
|
|
|
cache = subprocess.check_output(
|
|
|
|
|
[str(binaries["npm"]), "config", "get", "cache"],
|
|
|
|
|
env=environment, text=True, encoding="utf-8", timeout=60,
|
|
|
|
|
).strip()
|
|
|
|
|
outputs["npm-cache-path"] = cache
|
|
|
|
|
file_commands("GITHUB_ENV", exported)
|
|
|
|
|
file_commands("GITHUB_OUTPUT", outputs)
|
|
|
|
|
add_path(path)
|
|
|
|
|
print("PM toolchain ready: " + ", ".join(f"{name} {facts.get(name)['version']}" for name in names))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def dependencies(args) -> None:
|
2026-09-23 18:13:38 -04:00
|
|
|
if args.extras is None and not args.test_environment:
|
2026-09-08 12:45:15 -04:00
|
|
|
return
|
|
|
|
|
import tomllib
|
|
|
|
|
|
2026-09-18 20:02:36 -04:00
|
|
|
from pm.environments import selected_venv
|
2026-09-23 18:13:38 -04:00
|
|
|
from pm import check_project_lock, sync_venv
|
refactor(pm): one plugin-input request for sync_venv and venv_is_current
sync_venv took four mutually exclusive plugin kwargs (plugin_dirs,
extra_plugin_dirs, selection, staged_plugin) and venv_is_current two, with
runtime ValueErrors guarding the combinations. Replace them with a single
`plugins=` argument typed as one of pm.plugin_inputs.Members, Candidates,
Selection or StagedUpdate, so a conflicting request cannot be expressed.
The module also owns the worker wire encoding that client.py and worker.py
each duplicated.
pm.install.sync_venv is split into cohesive helpers (feature policy,
install lock, publication snapshot, target selection, commit) with the
same ordering, receipts and recovery; its CC drops from 54 to 16.
All in-tree callers and tests move to the new argument.
2026-09-24 13:58:29 -04:00
|
|
|
from pm.plugin_inputs import Members
|
2026-09-08 12:45:15 -04:00
|
|
|
from pm.paths import repo_root
|
|
|
|
|
|
|
|
|
|
project = repo_root()
|
|
|
|
|
metadata = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8-sig"))
|
2026-09-23 18:13:38 -04:00
|
|
|
extras = args.extras or []
|
|
|
|
|
unknown = set(extras) - metadata["project"].get("optional-dependencies", {}).keys()
|
2026-09-08 12:45:15 -04:00
|
|
|
if unknown:
|
|
|
|
|
raise ValueError(f"unknown project extras: {sorted(unknown)}")
|
2026-09-11 18:14:43 -04:00
|
|
|
# Frozen sync must not turn a stale project lock into a green job.
|
|
|
|
|
check_project_lock(project, explicit=True)
|
2026-09-23 18:13:38 -04:00
|
|
|
if args.test_environment:
|
|
|
|
|
from pm.testenv import ensure_testenv
|
|
|
|
|
|
|
|
|
|
# CI and activation select the same side environment, never PM facts.
|
|
|
|
|
python = ensure_testenv(project, extras)
|
|
|
|
|
venv = python.parent.parent
|
2026-09-09 15:17:08 -04:00
|
|
|
else:
|
refactor(pm): one plugin-input request for sync_venv and venv_is_current
sync_venv took four mutually exclusive plugin kwargs (plugin_dirs,
extra_plugin_dirs, selection, staged_plugin) and venv_is_current two, with
runtime ValueErrors guarding the combinations. Replace them with a single
`plugins=` argument typed as one of pm.plugin_inputs.Members, Candidates,
Selection or StagedUpdate, so a conflicting request cannot be expressed.
The module also owns the worker wire encoding that client.py and worker.py
each duplicated.
pm.install.sync_venv is split into cohesive helpers (feature policy,
install lock, publication snapshot, target selection, commit) with the
same ordering, receipts and recovery; its CC drops from 54 to 16.
All in-tree callers and tests move to the new argument.
2026-09-24 13:58:29 -04:00
|
|
|
sync_venv(extras, explicit=True, plugins=Members([]))
|
2026-09-09 15:17:08 -04:00
|
|
|
venv = selected_venv(project)
|
2026-09-08 12:45:15 -04:00
|
|
|
bindir = venv / ("Scripts" if os.name == "nt" else "bin")
|
|
|
|
|
python = bindir / ("python.exe" if os.name == "nt" else "python")
|
|
|
|
|
python3_alias(python)
|
|
|
|
|
file_commands("GITHUB_ENV", {
|
|
|
|
|
"HERMES_PYTHON": python,
|
|
|
|
|
"VIRTUAL_ENV": venv,
|
|
|
|
|
"PYTHONPATH": project,
|
|
|
|
|
})
|
|
|
|
|
file_commands("GITHUB_OUTPUT", {"python-path": python, "venv": venv})
|
|
|
|
|
add_path([str(bindir)])
|
|
|
|
|
print(f"PM dependencies ready: {args.extras} in {venv}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
fix(pm): preserve pinned binary inputs in R2
Termux removes old package files, so a pinned URL and hash do not keep
build inputs available. Preserve the exact bytes without changing pins.
Archive every PM HTTP artifact and the Termux runtime inputs by SHA256.
CI reads R2 first. Only a missing object permits an upstream download,
hash verification, immutable upload, and verified readback. Seed the
actual toolchain and payload stores before their consumers run.
Use the public archive as a pinned fallback in PM, bootstrap installers,
and Nix fetchers. Keep network retries bounded and report attempted URLs.
Keep publication credentials in protected CI jobs, not installed clients.
Verification:
- 283 targeted tests passed; five POSIX tests skipped on Windows.
- All 87 preserved Termux packages passed local archive miss/hit checks.
- Native ARM64 ripgrep installed through the mirror and ran successfully.
- Wheel import, workflow lint, Python lint, shell syntax, and pins passed.
Live R2 publication, POSIX tests, and Nix builds remain for native CI.
The real-byte archive checks used loopback HTTP, not the live bucket.
2026-09-10 18:17:08 -04:00
|
|
|
phases = {"prepare": prepare, "archive-inputs": archive_inputs, "install": install, "dependencies": dependencies}
|
2026-09-08 12:45:15 -04:00
|
|
|
parser.add_argument("phase", choices=list(phases))
|
|
|
|
|
parser.add_argument("--toolchain", choices=["python", "node", "all"], default="python")
|
|
|
|
|
parser.add_argument("--home", type=Path, required=True)
|
|
|
|
|
parser.add_argument("--extras", type=parse_extras, default="")
|
2026-09-23 18:13:38 -04:00
|
|
|
parser.add_argument("--test-environment", action="store_true")
|
feat(ci): pm-locked ffmpeg + minimal chat driver deps for native smoke legs
The e2e-screen-record action installed ffmpeg through three different
OS package managers (apt, brew, winget). winget is the flaky leg on
windows-11-arm and serves an x64-gyan build that runs emulated on ARM;
choco's community package wraps the same gyan x64 zip, so a choco
fallback would not fix either problem. PM already ships a locked,
sha256-pinned ffmpeg (martin-riedl posix, BtbN win32 including a NATIVE
winarm64 build), so the recording action now verifies ffmpeg on PATH
instead of installing it, and the pinned binary rides the same
tools-cache as node/python.
- setup-pm learns a `packages` input (extra PM tools beyond the
toolchain roots, e.g. ffmpeg) threaded through setup_toolchain.py's
prepare/install/archive-inputs phases; the tools-cache key gains an
extra-packages fragment so existing keys stay byte-identical.
- The six chat-driver jobs pass `packages: ffmpeg` to setup-pm.
- e2e-screen-record drops the apt/brew/winget install steps, the
ffmpeg actions/cache steps and the save-cache input; Xvfb (headless
linux) and the macOS replayd-approval hack stay.
- The "Install locked chat driver dependencies" step installs only the
tests-js workspace with --omit=dev instead of the whole apps/desktop
tree: the drivers need @playwright/test, zod (previously a phantom
hoisted from @assistant-ui/react), js-yaml and semver only. 64 pure
packages in ~2s vs ~1800 including electron-builder and native
builds; the tree-shaken node_modules runs the real driver modules
(verified by importing desktop-chat-smoke.ts and update-window-chat
end to end).
- tests pin the new contracts; tests/install/README.md updated.
2026-09-14 13:35:13 -04:00
|
|
|
parser.add_argument("--packages", type=parse_package_list, default=[], help="extra PM tools beyond the toolchain roots (e.g. ffmpeg)")
|
2026-09-08 12:45:15 -04:00
|
|
|
args = parser.parse_args()
|
2026-09-23 18:13:38 -04:00
|
|
|
if args.toolchain == "node" and (args.extras is not None or args.test_environment):
|
|
|
|
|
parser.error("Python dependencies require the python or all toolchain")
|
2026-09-08 12:45:15 -04:00
|
|
|
os.environ["HERMES_HOME"] = str(args.home.resolve())
|
|
|
|
|
os.environ["HERMES_RUNTIME_DIR"] = str(args.home.resolve() / "tools")
|
|
|
|
|
phases[args.phase](args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|