Compare commits

...

3 Commits

Author SHA1 Message Date
Eitan Yarmush
5d333f4ec1 feat: add ack-based bind notification for deterministic listener readiness
Replace the broadcast channel in BindStore with an mpsc channel that
carries an optional oneshot::Sender for acknowledgment. Callers of
insert_bind_with_ack() receive a oneshot::Receiver that resolves once
the TCP listener is actually bound, eliminating the race between bind
registration and listener readiness.

Previously, insert_bind() fired-and-forgot via broadcast — callers had
no way to know when the listener was ready to accept connections. This
caused intermittent connection timeouts in moat's DNAT proxy setup where
the gateway bind was registered but the TCP listener hadn't started yet.

The existing insert_bind() API is preserved as a convenience wrapper
that passes ack=None, so all current callers (XDS sync, local config
reload) are unaffected.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Eitan Yarmush <eitan.yarmush@solo.io>
2026-03-30 17:44:21 +00:00
Eitan Yarmush
7ef1e591ba Merge branch 'main' of github.com:agentgateway/agentgateway into eitanya/bind-ack 2026-03-30 17:38:35 +00:00
Eitan Yarmush
b8b4042a69 fix: use protocol-filtered matching in detect_misdirected
detect_misdirected() used unfiltered best_match() to verify the
TLS-selected listener, which could return an HTTP listener with the
same wildcard hostname. Since ListenerSet uses HashMap (non-deterministic
iteration), this caused spurious 421 errors when BindProtocol::auto
serves both HTTP and HTTPS listeners on the same bind.

Use best_match_tls() instead, matching the protocol-aware pattern
already used in TLS handshake listener selection (gateway.rs) and
the HTTP fallback path (best_match_http).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Eitan Yarmush <eitan.yarmush@solo.io>
2026-03-28 10:55:39 +00:00
8 changed files with 180 additions and 95 deletions

View File

@ -77,10 +77,11 @@ pub async fn run(config: Arc<Config>) -> anyhow::Result<Bound> {
);
let (xds_tx, xds_rx) = tokio::sync::watch::channel(());
let state_mgr =
let mut state_mgr =
state_manager::StateManager::new(config.clone(), control_client.clone(), xds_metrics, xds_tx)
.await?;
let stores = state_mgr.stores();
let bind_rx = state_mgr.take_bind_rx();
let mut xds_rx_for_task = xds_rx.clone();
tokio::spawn(async move {
@ -116,7 +117,7 @@ pub async fn run(config: Arc<Config>) -> anyhow::Result<Bound> {
mcp_state: mcp::App::new(stores.clone(), config.session_encoder.clone()),
};
let gw = proxy::Gateway::new(Arc::new(pi), drain_rx.clone());
let gw = proxy::Gateway::new(Arc::new(pi), drain_rx.clone(), bind_rx);
// Run the agentgateway in the data plane worker pool.
let mut xds_rx_for_proxy = xds_rx.clone();

View File

@ -21,7 +21,7 @@ use crate::types::discovery::{
use crate::*;
fn run_test(req: &Request, routes: &[(&str, Vec<&str>, Vec<RouteMatch>)]) -> Option<String> {
let stores = Stores::with_ipv6_enabled(true);
let stores = Stores::default();
let network = strng::literal!("network");
let dummy_dest = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1000);
@ -1548,7 +1548,7 @@ fn bench(b: Bencher, (host, route): (u64, u64)) {
.collect(),
),
});
let stores = Stores::with_ipv6_enabled(true);
let stores = Stores::default();
let network = strng::literal!("network");
let dummy_dest = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1000);
let req = request("http://example.com", http::Method::GET, &[]);

View File

@ -31,7 +31,7 @@ async fn setup_with_prefix(prefix: &str) -> (MockServer, Handler) {
let parsed = reqwest::Url::parse(&host).unwrap();
let config = crate::config::parse_config("{}".to_string(), None).unwrap();
let encoder = config.session_encoder.clone();
let stores = Stores::with_ipv6_enabled(config.ipv6_enabled);
let stores = Stores::default();
let client = Client::new(
&client::Config {
resolver_cfg: ResolverConfig::default(),

View File

@ -17,7 +17,6 @@ use rand::RngExt;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::watch;
use tokio::task::{AbortHandle, JoinSet};
use tokio_stream::StreamExt;
use tracing::{Instrument, debug, error, event, info, info_span, warn};
use agent_core::strng;
@ -127,84 +126,100 @@ impl TryFrom<&http::Uri> for HboneAddress {
pub struct Gateway {
pi: Arc<ProxyInputs>,
drain: drain::DrainWatcher,
bind_rx: tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>,
}
impl Gateway {
pub fn new(pi: Arc<ProxyInputs>, drain: DrainWatcher) -> Gateway {
Gateway { drain, pi }
pub fn new(
pi: Arc<ProxyInputs>,
drain: DrainWatcher,
bind_rx: tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>,
) -> Gateway {
Gateway { drain, pi, bind_rx }
}
pub async fn run(self) {
pub async fn run(mut self) {
let drain = self.drain.clone();
let subdrain = self.drain.clone();
let mut js = JoinSet::new();
let (initial_binds, mut binds) = {
let initial_binds = {
let binds = self.pi.stores.read_binds();
(binds.all(), binds.subscribe())
binds.all()
};
let mut active: HashMap<SocketAddr, AbortHandle> = HashMap::new();
let mut handle_bind = |js: &mut JoinSet<anyhow::Result<()>>, b: Event<Arc<Bind>>| {
let b = match b {
Event::Add(b) => b,
Event::Remove(to_remove) => {
if let Some(h) = active.remove(&to_remove.address) {
h.abort();
let mut handle_bind =
|js: &mut JoinSet<anyhow::Result<()>>,
b: Event<Arc<Bind>>,
ack: Option<tokio::sync::oneshot::Sender<()>>| {
let b = match b {
Event::Add(b) => b,
Event::Remove(to_remove) => {
if let Some(h) = active.remove(&to_remove.address) {
h.abort();
}
return;
},
};
if active.contains_key(&b.address) {
debug!("bind already exists");
// Still ack — the bind address is already listening.
if let Some(tx) = ack {
let _ = tx.send(());
}
return;
},
};
if active.contains_key(&b.address) {
debug!("bind already exists");
return;
}
}
debug!("add bind {}", b.address);
if self.pi.cfg.threading_mode == crate::ThreadingMode::ThreadPerCore {
let core_ids = core_affinity::get_core_ids().unwrap();
let _ = core_ids
.into_iter()
.map(|id| {
let subdrain = subdrain.clone();
let pi = self.pi.clone();
let b = b.clone();
std::thread::spawn(move || {
let res = core_affinity::set_for_current(id);
if !res {
panic!("failed to set current CPU")
}
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
let _ = Self::run_bind(pi.clone(), subdrain.clone(), b.clone())
.in_current_span()
.await;
})
debug!("add bind {}", b.address);
if self.pi.cfg.threading_mode == crate::ThreadingMode::ThreadPerCore {
let core_ids = core_affinity::get_core_ids().unwrap();
let _ = core_ids
.into_iter()
.map(|id| {
let subdrain = subdrain.clone();
let pi = self.pi.clone();
let b = b.clone();
std::thread::spawn(move || {
let res = core_affinity::set_for_current(id);
if !res {
panic!("failed to set current CPU")
}
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
let _ =
Self::run_bind(pi.clone(), subdrain.clone(), b.clone(), None)
.in_current_span()
.await;
})
})
})
})
.collect::<Vec<_>>();
} else {
let task =
js.spawn(Self::run_bind(self.pi.clone(), subdrain.clone(), b.clone()).in_current_span());
active.insert(b.address, task);
}
};
.collect::<Vec<_>>();
// ThreadPerCore spawns OS threads — we can't easily plumb the ack
// through all of them. Send the ack now; in practice ThreadPerCore
// binds synchronously before spawning the accept loop.
if let Some(tx) = ack {
let _ = tx.send(());
}
} else {
let task = js.spawn(
Self::run_bind(self.pi.clone(), subdrain.clone(), b.clone(), ack)
.in_current_span(),
);
active.insert(b.address, task);
}
};
for bind in initial_binds {
handle_bind(&mut js, Event::Add(bind))
handle_bind(&mut js, Event::Add(bind), None)
}
let wait = drain.wait_for_drain();
tokio::pin!(wait);
loop {
tokio::select! {
Some(res) = binds.next() => {
let Ok(res) = res else {
// TODO: move to unbuffered
warn!("lagged on bind update");
continue;
};
handle_bind(&mut js, res);
Some(bind_event) = self.bind_rx.recv() => {
handle_bind(&mut js, bind_event.event, bind_event.ack);
}
Some(res) = js.join_next() => {
warn!("bind complete {res:?}");
@ -225,6 +240,7 @@ impl Gateway {
pi: Arc<ProxyInputs>,
drain: DrainWatcher,
b: Arc<Bind>,
ack: Option<tokio::sync::oneshot::Sender<()>>,
) -> anyhow::Result<()> {
let min_deadline = pi.cfg.termination_min_deadline;
let max_deadline = pi.cfg.termination_max_deadline;
@ -257,6 +273,10 @@ impl Gateway {
} else {
(pi, TcpListener::bind(b.address).await?)
};
// Listener is bound — notify the caller that it's safe to send traffic.
if let Some(tx) = ack {
let _ = tx.send(());
}
info!(bind = name.as_str(), "started bind");
let component = format!("bind {name}");

View File

@ -19,6 +19,9 @@ pub struct StateManager {
#[serde(skip_serializing)]
xds_client: Option<agent_xds::AdsClient>,
#[serde(skip_serializing)]
bind_rx: Option<tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>>,
}
pub const ADDRESS_TYPE: Strng = strng::literal!("type.googleapis.com/istio.workload.Address");
@ -35,7 +38,8 @@ impl StateManager {
awaiting_ready: tokio::sync::watch::Sender<()>,
) -> anyhow::Result<Self> {
let xds = &config.xds;
let stores = Stores::with_ipv6_enabled(config.ipv6_enabled);
let crate::store::StoresWithBindRx { stores, bind_rx } =
Stores::with_ipv6_enabled(config.ipv6_enabled);
let xds_client = if let Some(addr) = &xds.address {
let connector = control::grpc_connector(
client.clone(),
@ -72,13 +76,27 @@ impl StateManager {
};
Box::pin(local_client.run()).await?;
}
Ok(Self { stores, xds_client })
Ok(Self {
stores,
xds_client,
bind_rx: Some(bind_rx),
})
}
pub fn stores(&self) -> Stores {
self.stores.clone()
}
/// Take the bind event receiver. Must be called exactly once before
/// passing it to [`crate::proxy::gateway::Gateway::new`].
pub fn take_bind_rx(
&mut self,
) -> tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent> {
self.bind_rx
.take()
.expect("bind_rx already taken or StateManager not properly initialized")
}
pub async fn run(self) -> anyhow::Result<()> {
match self.xds_client {
Some(xds) => xds.run().await.map_err(|e| anyhow::anyhow!(e)),

View File

@ -2,9 +2,7 @@ use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use agent_xds::{RejectedConfig, XdsUpdate};
use futures_core::Stream;
use itertools::Itertools;
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
use tracing::{Level, instrument};
use crate::cel::ContextBuilder;
@ -62,7 +60,7 @@ pub struct Store {
service_routes: HashMap<NamespacedHostname, RouteSet>,
service_tcp_routes: HashMap<NamespacedHostname, TCPRouteSet>,
tx: tokio::sync::broadcast::Sender<Event<Arc<Bind>>>,
tx: tokio::sync::mpsc::UnboundedSender<crate::store::BindEvent>,
}
#[derive(Default, Debug, Clone)]
@ -369,9 +367,11 @@ pub struct LLMResponsePolicies {
pub prompt_guard: Vec<ResponseGuard>,
}
impl Default for Store {
fn default() -> Self {
Self::with_ipv6_enabled(true)
impl Store {
/// Create a store with a disconnected channel (for tests that don't need bind events).
pub fn new_disconnected(ipv6_enabled: bool) -> Self {
let (store, _rx) = Self::with_ipv6_enabled(ipv6_enabled);
store
}
}
@ -383,9 +383,11 @@ pub struct RoutePath<'a> {
}
impl Store {
pub fn with_ipv6_enabled(ipv6_enabled: bool) -> Self {
let (tx, _) = tokio::sync::broadcast::channel(1000);
Self {
pub fn with_ipv6_enabled(
ipv6_enabled: bool,
) -> (Self, tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>) {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let store = Self {
ipv6_enabled,
binds: Default::default(),
resources: Default::default(),
@ -398,13 +400,8 @@ impl Store {
service_routes: Default::default(),
service_tcp_routes: Default::default(),
tx,
}
}
pub fn subscribe(
&self,
) -> impl Stream<Item = Result<Event<Arc<Bind>>, BroadcastStreamRecvError>> + use<> {
let sub = self.tx.subscribe();
tokio_stream::wrappers::BroadcastStream::new(sub)
};
(store, rx)
}
pub fn route_policies(&self, path: &RoutePath<'_>, inline: &[TrafficPolicy]) -> RoutePolicies {
@ -811,7 +808,10 @@ impl Store {
)]
pub fn remove_bind(&mut self, bind: BindKey) {
if let Some(old) = self.binds.remove(&bind) {
let _ = self.tx.send(Event::Remove(old));
let _ = self.tx.send(crate::store::BindEvent {
event: Event::Remove(old),
ack: None,
});
}
}
#[instrument(
@ -911,7 +911,24 @@ impl Store {
skip_all,
fields(bind=%bind.key),
)]
pub fn insert_bind(&mut self, mut bind: Bind) {
pub fn insert_bind(&mut self, bind: Bind) {
self.insert_bind_with_ack(bind, None);
}
/// Insert a bind and return a [`oneshot::Receiver`] that resolves once the
/// TCP listener is actually bound. Use this when you need to block until
/// the listener is ready (e.g. before telling a sandbox it can send traffic).
#[instrument(
level = Level::INFO,
name="insert_bind_with_ack",
skip_all,
fields(bind=%bind.key),
)]
pub fn insert_bind_with_ack(
&mut self,
mut bind: Bind,
ack: Option<tokio::sync::oneshot::Sender<()>>,
) {
debug!(bind=%bind.key, "insert bind");
// Insert any staged listeners
@ -934,8 +951,10 @@ impl Store {
}
let arc = Arc::new(bind);
self.binds.insert(arc.key.clone(), arc.clone());
// ok to have no subs
let _ = self.tx.send(Event::Add(arc));
let _ = self.tx.send(crate::store::BindEvent {
event: Event::Add(arc),
ack,
});
}
pub fn insert_backend(&mut self, key: BackendKey, b: BackendWithPolicies) {
@ -1508,7 +1527,7 @@ mod tests {
#[test]
fn route_policies_are_kind_scoped() {
let mut store = Store::default();
let mut store = Store::new_disconnected(true);
let listener = listener();
let http_route = route("r", "ns", Some("HTTPRoute"));
@ -1539,7 +1558,7 @@ mod tests {
/// Tests that frontend policies at listener level take precedence over gateway level policies
#[test]
fn frontend_policy_listener_precedence() {
let mut store = Store::default();
let mut store = Store::new_disconnected(true);
let listener = listener();
// Insert both gateway and listener level frontend policies
@ -1566,7 +1585,7 @@ mod tests {
#[test]
fn frontend_network_authorization_policies_merge() {
let mut store = Store::default();
let mut store = Store::new_disconnected(true);
let listener = listener();
insert_gateway_level_network_authorization_policy(
&mut store,

View File

@ -24,26 +24,52 @@ pub enum Event<T> {
Remove(T),
}
/// A bind event with an optional acknowledgment channel.
///
/// When `ack` is `Some`, the consumer should send `()` on it after the TCP
/// listener is actually bound. This lets callers of `insert_bind` wait for
/// the listener to be ready instead of racing.
pub struct BindEvent {
pub event: Event<Arc<crate::types::agent::Bind>>,
pub ack: Option<tokio::sync::oneshot::Sender<()>>,
}
impl std::fmt::Debug for BindEvent {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BindEvent")
.field("event", &self.event)
.field("has_ack", &self.ack.is_some())
.finish()
}
}
#[derive(Clone, Debug)]
pub struct Stores {
pub discovery: discovery::StoreUpdater,
pub binds: binds::StoreUpdater,
}
/// Return type from [`Stores::with_ipv6_enabled`] — the stores plus the bind
/// event receiver that [`crate::proxy::gateway::Gateway`] must consume.
pub struct StoresWithBindRx {
pub stores: Stores,
pub bind_rx: tokio::sync::mpsc::UnboundedReceiver<BindEvent>,
}
impl Default for Stores {
fn default() -> Self {
Self::with_ipv6_enabled(true)
Self::with_ipv6_enabled(true).stores
}
}
impl Stores {
pub fn with_ipv6_enabled(ipv6_enabled: bool) -> Stores {
Stores {
pub fn with_ipv6_enabled(ipv6_enabled: bool) -> StoresWithBindRx {
let (bind_store, bind_rx) = binds::Store::with_ipv6_enabled(ipv6_enabled);
let stores = Stores {
discovery: discovery::StoreUpdater::new(Arc::new(RwLock::new(discovery::Store::new()))),
binds: binds::StoreUpdater::new(Arc::new(RwLock::new(binds::Store::with_ipv6_enabled(
ipv6_enabled,
)))),
}
binds: binds::StoreUpdater::new(Arc::new(RwLock::new(bind_store))),
};
StoresWithBindRx { stores, bind_rx }
}
pub fn read_binds(&self) -> std::sync::RwLockReadGuard<'_, store::BindStore> {
self.binds.read()

View File

@ -735,7 +735,8 @@ pub fn setup_proxy_test(cfg: &str) -> anyhow::Result<TestBind> {
agent_core::telemetry::testing::setup_test_logging();
let config = crate::config::parse_config(cfg.to_string(), None)?;
let encoder = config.session_encoder.clone();
let stores = Stores::with_ipv6_enabled(config.ipv6_enabled);
let crate::store::StoresWithBindRx { stores, bind_rx: _bind_rx } =
Stores::with_ipv6_enabled(config.ipv6_enabled);
let client = client::Client::new(&config.dns, None, Default::default(), None);
let (drain_tx, drain_rx) = drain::new();
let pi = Arc::new(ProxyInputs {