Teknium f7dde7f56d fix(web): web_extract reports a target's HTTP 4xx/5xx as an error on every extract provider
Scraping vendors answer 200 and report the target page's own status inside the
payload. Firecrawl (direct key, Nous Tool Gateway, keyless REST) put it in
metadata.status_code / statusCode and Hermes never read it, so
web_extract("https://httpbin.org/status/403") returned the stub error page
with error: null: the agent could not tell blocked from blank, the keyless
rescue never ran, and the error page was cached as content.

One shared helper in plugins/web/_common.py (origin_http_error /
with_origin_status / vendor_error_entry) now turns an origin status >= 400
into "HTTP <code> from origin[: reason]", keeping any error-page body as
content. Applied to:
- Firecrawl SDK/gateway scrape and the keyless REST scrape (both key spellings)
- Parallel keyed + keyless (error was the bare "http_error"; the payload's
  http_status_code was dropped)
- Exa keyed (URLs Exa could not fetch were silently missing from results;
  their reason lives in response.statuses)

Tavily, Perplexity and Keenable already surface per-URL failures (Tavily's
failed_results carry no status code).

Salvages the Firecrawl statusCode check from #99549 (and #99556), which read
only the camelCase key and only flagged empty content, so it missed the
Gateway/SDK case where the 403 body is a non-empty stub page.

Found on Hermes Cloud staging.

Co-authored-by: liuhao1024 <sunsky.lau@gmail.com>
Co-authored-by: Ahmett101 <ahmet.tunc@gmail.com>
2026-10-28 13:13:12 -07:00

91 lines
3.9 KiB
Python

"""Parallel.ai web search (sync ``Parallel`` SDK) + async extract (``AsyncParallel``).
Env: ``PARALLEL_API_KEY`` (https://parallel.ai), optional
``PARALLEL_SEARCH_MODE`` = agentic (default) | fast | one-shot.
"""
from __future__ import annotations
import asyncio
import logging
import os
from typing import Any, Dict, List
from plugins.web._common import (
SEARCH_LIMIT_CAP, BaseWebSearchProvider, cached_sdk_client, document, keyless_extract, keyless_search,
keyless_variant_schema, provider_env, run_extract_async, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,
)
logger = logging.getLogger(__name__)
_MISSING_KEY = "PARALLEL_API_KEY environment variable not set. Get your API key at https://parallel.ai"
def _client(slot: str, cls_name: str) -> Any:
def _factory(api_key: str) -> Any:
import parallel # deliberately lazy
return getattr(parallel, cls_name)(api_key=api_key)
# The key is checked before the SDK is ensured (cached_sdk_client): without a key the
# error names the key, not an install the user may not be allowed to run.
return cached_sdk_client(slot, "PARALLEL_API_KEY", _MISSING_KEY, "parallel-web", _factory)
def _get_sync_client() -> Any:
return _client("_parallel_client", "Parallel")
def _get_async_client() -> Any:
return _client("_async_parallel_client", "AsyncParallel")
def _resolve_search_mode() -> str:
mode = os.getenv("PARALLEL_SEARCH_MODE", "agentic").lower().strip()
return mode if mode in {"fast", "one-shot", "agentic"} else "agentic"
class ParallelWebSearchProvider(BaseWebSearchProvider):
"""Parallel.ai search + async extract provider."""
NAME = "parallel"
DISPLAY_NAME = "Parallel"
KEY_ENV = "PARALLEL_API_KEY"
EXTRACT = True
KEYLESS = True
def search(self, query: str, limit: int = 5) -> dict[str, Any]:
def _body() -> dict[str, Any]:
if use_keyless("parallel", provider_env("PARALLEL_API_KEY")):
return keyless_search("Parallel", "parallel", query, limit, logger)
mode = _resolve_search_mode()
logger.info("Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit)
response = _get_sync_client().beta.search(search_queries=[query], objective=query, mode=mode, max_results=min(limit, SEARCH_LIMIT_CAP))
return search_ok([
web_hit(r.url or "", r.title or "", " ".join(r.excerpts or []), i + 1)
for i, r in enumerate(response.results or [])
])
return run_search("Parallel", logger, _body, sdk=True)
async def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
async def _body() -> list[dict[str, Any]]:
if use_keyless("parallel", provider_env("PARALLEL_API_KEY")):
# Keyless ring is blocking HTTP — hop off the event loop.
return await asyncio.to_thread(keyless_extract, "Parallel", "parallel", urls, logger)
logger.info("Parallel extract: %d URL(s)", len(urls))
response = await _get_async_client().beta.extract(urls=urls, full_content=True)
results = [document(r.url or "", r.title or "", r.full_content or "\n\n".join(r.excerpts or [])) for r in response.results or []]
return results + [
vendor_error_entry(e.url or "", status=getattr(e, "http_status_code", None), reason=e.error_type, body=e.content)
for e in response.errors or []
]
return await run_extract_async("Parallel", logger, urls, _body, sdk=True)
def get_setup_schema(self) -> dict[str, Any]:
return keyless_variant_schema(
"Parallel", "PARALLEL_API_KEY", "https://parallel.ai",
free_tag="Objective-tuned search + page extraction on Parallel's anonymous free tier. Rate-limited under burst load.",
paid_tag="Objective-tuned search + parallel page extraction via the Parallel SDK. Unthrottled, guaranteed service.",
)