An OAuth refresh atomically replaces a token with same-size bytes within the mtime tick of the last sync; the (mtime, size) key matched, so the sandbox kept the stale credential. Key synced files on utils.file_signature (adds inode and ctime) so replaces are detected. A key whose mtime is still racy is recorded as never-matching, and the next sync confirms the bytes against the pushed hash: an in-place rewrite inside one coarse tick (ctime ticks just as coarsely) is re-uploaded, unchanged bytes are not. Planning sits inside the sync failure boundary: the content check reads the file, and one removed after its stat must fail the cycle (retried next time), not the terminal command that triggered the sync.
468 lines
16 KiB
Python
468 lines
16 KiB
Python
"""Tests for FileSyncManager — mtime tracking, deletion detection, transactional rollback."""
|
|
|
|
import concurrent.futures
|
|
import io
|
|
import os
|
|
import tarfile
|
|
import threading
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from tools.environments.file_sync import FileSyncManager, _FORCE_SYNC_ENV, iter_sync_files
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_files(tmp_path):
|
|
"""Create a few temp files to use as sync sources."""
|
|
files = {}
|
|
for name in ("cred_a.json", "cred_b.json", "skill_main.py"):
|
|
p = tmp_path / name
|
|
p.write_text(f"content of {name}", encoding="utf-8")
|
|
files[name] = str(p)
|
|
return files
|
|
|
|
|
|
def _make_get_files(tmp_files, remote_base="/root/.hermes"):
|
|
"""Return a get_files_fn that maps local files to remote paths."""
|
|
mapping = [(hp, f"{remote_base}/{name}") for name, hp in tmp_files.items()]
|
|
|
|
def get_files():
|
|
return [(hp, rp) for hp, rp in mapping if Path(hp).exists()]
|
|
|
|
return get_files
|
|
|
|
|
|
def _make_manager(tmp_files, remote_base="/root/.hermes", upload=None, delete=None):
|
|
"""Create a FileSyncManager with test callbacks."""
|
|
return FileSyncManager(
|
|
get_files_fn=_make_get_files(tmp_files, remote_base),
|
|
upload_fn=upload or MagicMock(),
|
|
delete_fn=delete or MagicMock(),
|
|
)
|
|
|
|
|
|
class TestMtimeSkip:
|
|
def test_unchanged_files_not_re_uploaded(self, tmp_files):
|
|
upload = MagicMock()
|
|
mgr = _make_manager(tmp_files, upload=upload)
|
|
|
|
mgr.sync(force=True)
|
|
assert upload.call_count == 3
|
|
|
|
upload.reset_mock()
|
|
mgr.sync(force=True)
|
|
assert upload.call_count == 0, "unchanged files should not be re-uploaded"
|
|
|
|
def test_same_size_in_place_rewrite_on_a_coarse_clock_is_re_uploaded(self, tmp_path, monkeypatch):
|
|
"""A filesystem whose timestamps did not tick between the sync and a rewrite (coarse mtime
|
|
AND ctime): a same-size in-place rewrite keeps every stat field, so a key recorded while its
|
|
mtime was that fresh must be confirmed by content before it is trusted."""
|
|
import pathlib
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
token = tmp_path / "token.json"
|
|
token.write_text('{"access":"AAAA"}', encoding="utf-8")
|
|
frozen_ns = time.time_ns()
|
|
real_stat = pathlib.Path.stat
|
|
|
|
def frozen_clock_stat(path, *args, **kwargs):
|
|
st = real_stat(path, *args, **kwargs)
|
|
return SimpleNamespace(st_mtime_ns=frozen_ns, st_ctime_ns=frozen_ns, st_size=st.st_size,
|
|
st_ino=st.st_ino, st_mode=st.st_mode, st_mtime=frozen_ns / 1e9)
|
|
|
|
monkeypatch.setattr(pathlib.Path, "stat", frozen_clock_stat)
|
|
uploaded = []
|
|
|
|
def bulk_upload(files):
|
|
uploaded.extend(Path(staged).read_text(encoding="utf-8") for staged, _remote in files)
|
|
|
|
mgr = FileSyncManager(
|
|
get_files_fn=lambda: [(str(token), "/root/.hermes/token.json")],
|
|
upload_fn=MagicMock(),
|
|
delete_fn=MagicMock(),
|
|
bulk_upload_fn=bulk_upload,
|
|
)
|
|
mgr.sync(force=True)
|
|
mgr.sync(force=True) # unchanged bytes: confirmed by content, not uploaded again
|
|
|
|
token.write_text('{"access":"BBBB"}', encoding="utf-8")
|
|
mgr.sync(force=True)
|
|
|
|
assert uploaded == ['{"access":"AAAA"}', '{"access":"BBBB"}']
|
|
|
|
@pytest.mark.platforms("linux", "macos")
|
|
def test_same_size_same_mtime_replace_is_re_uploaded(self, tmp_path):
|
|
"""An OAuth refresh atomically replaces a token with same-size bytes inside the mtime
|
|
tick of the last sync; the remote must still receive the new credential."""
|
|
token = tmp_path / "token.json"
|
|
pinned_ns = 1_800_000_000_000_000_000
|
|
token.write_text('{"access":"AAAA"}', encoding="utf-8")
|
|
os.utime(token, ns=(pinned_ns, pinned_ns))
|
|
uploaded = []
|
|
|
|
def bulk_upload(files):
|
|
uploaded.extend(Path(staged).read_text(encoding="utf-8") for staged, _remote in files)
|
|
|
|
mgr = FileSyncManager(
|
|
get_files_fn=lambda: [(str(token), "/root/.hermes/token.json")],
|
|
upload_fn=MagicMock(),
|
|
delete_fn=MagicMock(),
|
|
bulk_upload_fn=bulk_upload,
|
|
)
|
|
mgr.sync(force=True)
|
|
|
|
replacement = tmp_path / "token.json.tmp"
|
|
replacement.write_text('{"access":"BBBB"}', encoding="utf-8")
|
|
os.utime(replacement, ns=(pinned_ns, pinned_ns))
|
|
os.replace(replacement, token)
|
|
mgr.sync(force=True)
|
|
|
|
assert uploaded == ['{"access":"AAAA"}', '{"access":"BBBB"}']
|
|
|
|
|
|
def test_new_file_detected(self, tmp_files, tmp_path):
|
|
upload = MagicMock()
|
|
mgr = FileSyncManager(
|
|
get_files_fn=_make_get_files(tmp_files),
|
|
upload_fn=upload,
|
|
delete_fn=MagicMock(),
|
|
)
|
|
|
|
mgr.sync(force=True)
|
|
assert upload.call_count == 3
|
|
|
|
# Add a new file
|
|
new_file = tmp_path / "new_skill.py"
|
|
new_file.write_text("new content", encoding="utf-8")
|
|
tmp_files["new_skill.py"] = str(new_file)
|
|
# Recreate manager with updated file list
|
|
mgr._get_files_fn = _make_get_files(tmp_files)
|
|
|
|
upload.reset_mock()
|
|
mgr.sync(force=True)
|
|
assert upload.call_count == 1
|
|
|
|
|
|
class TestDeletion:
|
|
def test_removed_file_triggers_delete(self, tmp_files):
|
|
upload = MagicMock()
|
|
delete = MagicMock()
|
|
mgr = _make_manager(tmp_files, upload=upload, delete=delete)
|
|
|
|
mgr.sync(force=True)
|
|
delete.assert_not_called()
|
|
|
|
# Remove a file locally
|
|
os.unlink(tmp_files["cred_b.json"])
|
|
del tmp_files["cred_b.json"]
|
|
mgr._get_files_fn = _make_get_files(tmp_files)
|
|
|
|
mgr.sync(force=True)
|
|
delete.assert_called_once()
|
|
deleted_paths = delete.call_args[0][0]
|
|
assert any("cred_b.json" in p for p in deleted_paths)
|
|
|
|
|
|
|
|
class TestTransactionalRollback:
|
|
def test_upload_failure_rolls_back(self, tmp_files):
|
|
call_count = 0
|
|
|
|
def failing_upload(host_path, remote_path):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 2:
|
|
raise RuntimeError("upload failed")
|
|
|
|
mgr = _make_manager(tmp_files, upload=failing_upload)
|
|
|
|
# First sync fails (swallowed, logged, state rolled back)
|
|
mgr.sync(force=True)
|
|
|
|
# State should be empty (rolled back) — next sync retries all files
|
|
good_upload = MagicMock()
|
|
mgr._upload_fn = good_upload
|
|
mgr.sync(force=True)
|
|
assert good_upload.call_count == 3, "all files should be retried after rollback"
|
|
|
|
def test_delete_failure_rolls_back(self, tmp_files):
|
|
upload = MagicMock()
|
|
mgr = _make_manager(tmp_files, upload=upload)
|
|
|
|
# Initial sync
|
|
mgr.sync(force=True)
|
|
|
|
# Remove a file
|
|
os.unlink(tmp_files["skill_main.py"])
|
|
del tmp_files["skill_main.py"]
|
|
mgr._get_files_fn = _make_get_files(tmp_files)
|
|
|
|
# Delete fails (swallowed, state rolled back)
|
|
mgr._delete_fn = MagicMock(side_effect=RuntimeError("delete failed"))
|
|
mgr.sync(force=True)
|
|
|
|
# Next sync should retry the delete
|
|
good_delete = MagicMock()
|
|
mgr._delete_fn = good_delete
|
|
upload.reset_mock()
|
|
mgr.sync(force=True)
|
|
good_delete.assert_called_once()
|
|
|
|
def test_a_file_vanishing_mid_plan_is_retried_not_raised(self, tmp_files, monkeypatch):
|
|
"""sync() runs before every terminal command: a file removed between its stat and the
|
|
content check (a credential being rotated) must fail the cycle quietly, not the command."""
|
|
import tools.environments.file_sync as file_sync_mod
|
|
|
|
upload = MagicMock()
|
|
mgr = _make_manager(tmp_files, upload=upload)
|
|
mgr.sync(force=True)
|
|
# Re-stat as changed so the planner content-checks the already-pushed file.
|
|
mgr._synced_files = {remote: (-2, -2, -2, -2) for remote in mgr._synced_files}
|
|
real_sha = file_sync_mod._sha256_file
|
|
|
|
def vanished(path):
|
|
raise FileNotFoundError(path)
|
|
|
|
monkeypatch.setattr(file_sync_mod, "_sha256_file", vanished)
|
|
mgr.sync(force=True) # must not raise
|
|
|
|
monkeypatch.setattr(file_sync_mod, "_sha256_file", real_sha)
|
|
upload.reset_mock()
|
|
mgr.sync(force=True)
|
|
assert upload.call_count == 0, "the retry confirms the unchanged bytes instead of re-uploading"
|
|
|
|
|
|
class TestRateLimiting:
|
|
def test_sync_skipped_within_interval(self, tmp_files):
|
|
upload = MagicMock()
|
|
mgr = FileSyncManager(
|
|
get_files_fn=_make_get_files(tmp_files),
|
|
upload_fn=upload,
|
|
delete_fn=MagicMock(),
|
|
sync_interval=10.0,
|
|
)
|
|
|
|
mgr.sync(force=True)
|
|
assert upload.call_count == 3
|
|
|
|
upload.reset_mock()
|
|
# Without force, should skip due to rate limit
|
|
mgr.sync()
|
|
assert upload.call_count == 0
|
|
|
|
|
|
def test_env_var_forces_sync(self, tmp_files, tmp_path):
|
|
upload = MagicMock()
|
|
mgr = FileSyncManager(
|
|
get_files_fn=_make_get_files(tmp_files),
|
|
upload_fn=upload,
|
|
delete_fn=MagicMock(),
|
|
sync_interval=10.0,
|
|
)
|
|
|
|
mgr.sync(force=True)
|
|
upload.reset_mock()
|
|
|
|
new_file = tmp_path / "env_forced.txt"
|
|
new_file.write_text("env forced", encoding="utf-8")
|
|
tmp_files["env_forced.txt"] = str(new_file)
|
|
mgr._get_files_fn = _make_get_files(tmp_files)
|
|
|
|
with patch.dict(os.environ, {_FORCE_SYNC_ENV: "1"}):
|
|
mgr.sync()
|
|
assert upload.call_count == 1
|
|
|
|
def test_failed_sync_does_not_suppress_next_retry(self, tmp_files, monkeypatch):
|
|
"""A failed sync must not advance the rate-limit clock.
|
|
|
|
Regression: the failure path used to set ``_last_sync_time`` on
|
|
rollback, so the next non-forced ``sync()`` within ``sync_interval``
|
|
hit the rate-limit guard and returned early — silently suppressing the
|
|
retry the rollback had just prepared and leaving the remote stale.
|
|
"""
|
|
from tools.environments import file_sync
|
|
|
|
clock = {"t": 1000.0}
|
|
monkeypatch.setattr(file_sync, "_monotonic", lambda: clock["t"])
|
|
|
|
upload = MagicMock(side_effect=RuntimeError("transport down"))
|
|
mgr = FileSyncManager(
|
|
get_files_fn=_make_get_files(tmp_files),
|
|
upload_fn=upload,
|
|
delete_fn=MagicMock(),
|
|
sync_interval=10.0,
|
|
)
|
|
|
|
# First sync fails (forced bypasses the guard); state rolls back.
|
|
mgr.sync(force=True)
|
|
assert upload.call_count >= 1
|
|
|
|
# Transport recovers; advance the clock by LESS than the interval.
|
|
upload.reset_mock()
|
|
upload.side_effect = None
|
|
clock["t"] = 1002.0 # 2s later, < 10s interval
|
|
|
|
# The next non-forced cycle must retry, not be rate-limited away.
|
|
mgr.sync()
|
|
assert upload.call_count == 3, (
|
|
"a failed sync must not rate-limit the next retry"
|
|
)
|
|
|
|
|
|
class TestEdgeCases:
|
|
|
|
def test_file_disappears_between_list_and_upload(self, tmp_path):
|
|
"""File listed by get_files but deleted before _file_mtime_key reads it."""
|
|
f = tmp_path / "ephemeral.txt"
|
|
f.write_text("here now", encoding="utf-8")
|
|
|
|
upload = MagicMock()
|
|
mgr = FileSyncManager(
|
|
get_files_fn=lambda: [(str(f), "/root/.hermes/ephemeral.txt")],
|
|
upload_fn=upload,
|
|
delete_fn=MagicMock(),
|
|
)
|
|
|
|
# Delete the file before sync can stat it
|
|
os.unlink(str(f))
|
|
|
|
mgr.sync(force=True)
|
|
upload.assert_not_called() # _file_mtime_key returns None, skipped
|
|
|
|
|
|
class TestConcurrency:
|
|
@pytest.mark.platforms("linux")
|
|
def test_sync_back_waits_for_active_sync_transaction(self, tmp_path):
|
|
initial_file = tmp_path / "initial.png"
|
|
new_file = tmp_path / "new.png"
|
|
initial_file.write_bytes(b"initial")
|
|
upload_started = threading.Event()
|
|
release_upload = threading.Event()
|
|
sync_back_transport_started = threading.Event()
|
|
overlap_detected = threading.Event()
|
|
download_calls = []
|
|
|
|
def get_files():
|
|
return [
|
|
(str(path), f"/root/.hermes/cache/images/{path.name}")
|
|
for path in sorted(tmp_path.glob("*.png"))
|
|
]
|
|
|
|
def upload(_host_path, remote_path):
|
|
if remote_path == f"/root/.hermes/cache/images/{new_file.name}":
|
|
upload_started.set()
|
|
sync_back_transport_started.wait(timeout=1.0)
|
|
release_upload.set()
|
|
|
|
def bulk_download(destination):
|
|
if not release_upload.is_set():
|
|
overlap_detected.set()
|
|
sync_back_transport_started.set()
|
|
download_calls.append(destination)
|
|
with tarfile.open(destination, "w"):
|
|
pass
|
|
|
|
mgr = FileSyncManager(
|
|
get_files_fn=get_files,
|
|
upload_fn=upload,
|
|
delete_fn=MagicMock(),
|
|
bulk_download_fn=bulk_download,
|
|
)
|
|
mgr.sync(force=True)
|
|
new_file.write_bytes(b"new")
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
|
sync_future = executor.submit(mgr.sync, force=True)
|
|
assert upload_started.wait(timeout=2.0)
|
|
|
|
sync_back_future = executor.submit(mgr.sync_back, hermes_home=tmp_path)
|
|
|
|
sync_future.result(timeout=3.0)
|
|
sync_back_future.result(timeout=3.0)
|
|
|
|
assert len(download_calls) == 1
|
|
assert not overlap_detected.is_set()
|
|
|
|
|
|
class TestSyncBackSecurity:
|
|
@pytest.mark.platforms("linux")
|
|
def test_sync_back_does_not_overwrite_uploaded_credential_files(self, tmp_path, monkeypatch):
|
|
credential = tmp_path / "token.json"
|
|
credential.write_text("host-token", encoding="utf-8")
|
|
skill = tmp_path / "skill.py"
|
|
skill.write_text("host-skill", encoding="utf-8")
|
|
|
|
monkeypatch.setattr(
|
|
"tools.credential_files.get_credential_file_mounts",
|
|
lambda: [
|
|
{
|
|
"host_path": str(credential),
|
|
"container_path": "/root/.hermes/credentials/token.json",
|
|
}
|
|
],
|
|
)
|
|
monkeypatch.setattr(
|
|
"tools.credential_files.iter_skills_files",
|
|
lambda container_base="/root/.hermes": [
|
|
{
|
|
"host_path": str(skill),
|
|
"container_path": f"{container_base}/skills/skill.py",
|
|
}
|
|
],
|
|
)
|
|
monkeypatch.setattr(
|
|
"tools.credential_files.iter_cache_files",
|
|
lambda container_base="/root/.hermes": [],
|
|
)
|
|
|
|
def bulk_download(dest: Path) -> None:
|
|
with tarfile.open(dest, "w") as tar:
|
|
for name, data in {
|
|
"root/.hermes/credentials/token.json": b"remote-token",
|
|
"root/.hermes/skills/skill.py": b"remote-skill",
|
|
}.items():
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
|
|
mgr = FileSyncManager(
|
|
get_files_fn=lambda: iter_sync_files("/root/.hermes"),
|
|
upload_fn=MagicMock(),
|
|
delete_fn=MagicMock(),
|
|
bulk_download_fn=bulk_download,
|
|
)
|
|
|
|
mgr.sync(force=True)
|
|
mgr.sync_back(hermes_home=tmp_path)
|
|
|
|
assert credential.read_text(encoding="utf-8") == "host-token"
|
|
assert skill.read_text(encoding="utf-8") == "remote-skill"
|
|
|
|
|
|
class TestBulkUpload:
|
|
"""Tests for the optional bulk_upload_fn callback."""
|
|
|
|
|
|
|
|
def test_bulk_upload_rollback_on_failure(self, tmp_files):
|
|
"""Bulk upload failure rolls back synced state so next sync retries."""
|
|
bulk_upload = MagicMock(side_effect=RuntimeError("upload failed"))
|
|
mgr = FileSyncManager(
|
|
get_files_fn=_make_get_files(tmp_files),
|
|
upload_fn=MagicMock(),
|
|
delete_fn=MagicMock(),
|
|
bulk_upload_fn=bulk_upload,
|
|
)
|
|
|
|
mgr.sync(force=True) # fails, should rollback
|
|
|
|
# State rolled back: next sync should retry all files
|
|
bulk_upload.side_effect = None
|
|
bulk_upload.reset_mock()
|
|
mgr.sync(force=True)
|
|
bulk_upload.assert_called_once()
|
|
assert len(bulk_upload.call_args[0][0]) == 3
|