## Summary Host web_fetch previously sent every URL to a third-party reader and returned its errors even when the origin could be read directly. Structured feeds were converted into extracted text, and extractMode was echoed without controlling the output. - Read the origin through the existing guarded HTTP fetch before extraction. Preserve JSON, XML (including Atom and other +xml types), and plain text based on Content-Type, including endpoints without file extensions. - Stream direct responses with a 2 MiB body cap and a 10-second total deadline. Close responses on all exit paths, skip redirect/error and unsupported binary bodies, and request identity encoding to avoid unbounded decompression. Mark size-limited direct results as truncated. - Keep the configured reader, Jina by default, for HTML. Reuse the direct response when the reader returns an HTTP error, fails to connect, or cannot extract content. Keep account-level 401/402 pauses while allowing direct reads during the pause. - Remove the ineffective extractMode schema parameter and output field. Legacy keyword arguments remain accepted. Disclose third-party sharing of the full URL in the tool description. - Keep Serper/Jina defaults and the separate research implementation unchanged. ## Type - [x] Fix - [ ] Feature - [ ] Docs - [ ] CI / tooling - [ ] Refactor - [ ] Other ## Verification **Rebased**: head 007f972d2 on origin/main 8ee5ae0d8, without conflicts. `git range-diff` shows every commit replayed unchanged. The branch's own test files rerun at this head: 217 passed. Numbers elsewhere in this description are from earlier heads. - `uv run pytest tests/test_agent_tools_web_fetch.py tests/test_agent_tools_web_providers.py tests/test_security_web_ssrf.py tests/test_security_network.py tests/test_agent_loop_web_tools.py -q --disable-warnings`: 354 passed, 17 existing asyncio-mark warnings. - `uv run ruff check raven/agent/tools/web.py tests/test_agent_tools_web_fetch.py tests/test_agent_tools_web_providers.py tests/test_security_web_ssrf.py`: passed. - `uv run ruff format --check raven/agent/tools/web.py tests/test_agent_tools_web_fetch.py tests/test_agent_tools_web_providers.py tests/test_security_web_ssrf.py`: 4 files already formatted. - `make lint-types`: passed (the same all-extras type check used by CI). - `git diff --check`: passed. - [x] Relevant tests pass locally - [x] Relevant lint / type checks pass locally - [ ] User-facing docs or screenshots are updated when needed The model-facing tool description is updated; no UI or standalone documentation changes are included. Tests use mocked HTTP responses, not live provider credentials. ## Risk HTML reads now include an origin request before third-party extraction, adding latency and traffic; direct requests use a 10-second total deadline and retain at most 2 MiB of body data. Servers that ignore Accept-Encoding: identity are handled through the third-party reader instead of locally decompressing their bodies. Direct HTML fallback returns plain text, while structured data preserves its source text up to maxChars. Content routing depends on the origin's Content-Type header. Consumers must no longer rely on the removed extractMode output field; direct results report contentType and do not claim a finalUrl for pinned redirect responses. Direct requests reuse per-hop SSRF validation and DNS pinning and never carry vendor API keys. A successful direct data read sends no URL to the reader; HTML extraction still shares the full URL. Revert this change to restore the previous vendor-only path. - [x] Security impact considered - [x] Backward compatibility considered - [x] Rollback path is clear for risky changes ## Related Issues N/A --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
107 lines
3.7 KiB
Python
107 lines
3.7 KiB
Python
"""WebFetchTool routes URL validation through the strong SSRF check.
|
|
|
|
Mocks DNS so a public-looking hostname resolves to a private/internal IP
|
|
(the DNS-rebinding class the old scheme-only validator missed); the fetch
|
|
must refuse before any HTTP request.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from raven.agent.tools.web import WebFetchTool
|
|
|
|
|
|
def _resolve_to(monkeypatch: pytest.MonkeyPatch, ip: str) -> None:
|
|
def fake_getaddrinfo(host, *_a, **_k):
|
|
return [(0, 0, 0, "", (ip, 0))]
|
|
|
|
monkeypatch.setattr("socket.getaddrinfo", fake_getaddrinfo)
|
|
|
|
|
|
async def test_rejects_url_resolving_to_private_ip(monkeypatch):
|
|
_resolve_to(monkeypatch, "169.254.169.254") # cloud metadata endpoint
|
|
|
|
# If validation is bypassed this would attempt a real fetch; fail loudly.
|
|
def _boom(*_a, **_k):
|
|
raise AssertionError("HTTP client must not be constructed for a blocked URL")
|
|
|
|
monkeypatch.setattr("httpx.AsyncClient", _boom)
|
|
|
|
out = await WebFetchTool().execute(url="http://totally-public.example.com/x")
|
|
parsed = json.loads(out)
|
|
assert "validation failed" in parsed["error"]
|
|
# The reason sits in ``detail``: ``error`` is the tool-failure streak's
|
|
# classification key, so it carries the class and never the address.
|
|
assert "private/internal" in parsed["detail"]
|
|
|
|
|
|
async def test_rejects_loopback(monkeypatch):
|
|
def _boom(*_a, **_k):
|
|
raise AssertionError("HTTP client must not be constructed for a blocked URL")
|
|
|
|
monkeypatch.setattr("httpx.AsyncClient", _boom)
|
|
|
|
out = await WebFetchTool().execute(url="http://127.0.0.1/admin")
|
|
parsed = json.loads(out)
|
|
assert "validation failed" in parsed["error"]
|
|
|
|
|
|
async def test_rejects_non_http_scheme(monkeypatch):
|
|
out = await WebFetchTool().execute(url="file:///etc/passwd")
|
|
parsed = json.loads(out)
|
|
assert "validation failed" in parsed["error"]
|
|
|
|
|
|
@pytest.mark.parametrize("location", ["http://127.0.0.1/admin", "http://169.254.169.254/latest", "file:///etc/passwd"])
|
|
async def test_direct_fetch_checks_redirect_targets(monkeypatch, location):
|
|
_resolve_to(monkeypatch, "93.184.216.34")
|
|
calls = []
|
|
client_type = httpx.AsyncClient
|
|
|
|
def handle(request):
|
|
calls.append(request)
|
|
return httpx.Response(302, headers={"Location": location})
|
|
|
|
monkeypatch.setattr(
|
|
httpx,
|
|
"AsyncClient",
|
|
lambda **kwargs: client_type(**kwargs, transport=httpx.MockTransport(handle)),
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="Direct fetch blocked"):
|
|
await WebFetchTool()._direct_fetch("https://public.example/data")
|
|
|
|
assert len(calls) == 1
|
|
assert calls[0].url.host == "93.184.216.34"
|
|
assert calls[0].headers["host"] == "public.example"
|
|
assert calls[0].extensions["sni_hostname"] == "public.example"
|
|
|
|
|
|
async def test_direct_fetch_follows_public_redirect_and_preserves_json(monkeypatch):
|
|
_resolve_to(monkeypatch, "93.184.216.34")
|
|
calls = []
|
|
client_type = httpx.AsyncClient
|
|
|
|
def handle(request):
|
|
calls.append(request)
|
|
if request.url.path == "/start":
|
|
return httpx.Response(302, headers={"Location": "/data"})
|
|
return httpx.Response(200, text='{"items": [1, 2]}\n', headers={"Content-Type": "application/json"})
|
|
|
|
monkeypatch.setattr(
|
|
httpx,
|
|
"AsyncClient",
|
|
lambda **kwargs: client_type(**kwargs, transport=httpx.MockTransport(handle)),
|
|
)
|
|
|
|
result = json.loads(await WebFetchTool().execute("https://public.example/start"))
|
|
|
|
assert result["text"] == '{"items": [1, 2]}\n'
|
|
assert result["extractor"] == "direct-http"
|
|
assert [call.url.path for call in calls] == ["/start", "/data"]
|
|
assert all(call.url.host == "93.184.216.34" for call in calls)
|