Compare commits
3 Commits
main
...
eitanya/bi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5d333f4ec1 | ||
|
|
7ef1e591ba | ||
|
|
b8b4042a69 |
@ -77,10 +77,11 @@ pub async fn run(config: Arc<Config>) -> anyhow::Result<Bound> {
|
||||
);
|
||||
|
||||
let (xds_tx, xds_rx) = tokio::sync::watch::channel(());
|
||||
let state_mgr =
|
||||
let mut state_mgr =
|
||||
state_manager::StateManager::new(config.clone(), control_client.clone(), xds_metrics, xds_tx)
|
||||
.await?;
|
||||
let stores = state_mgr.stores();
|
||||
let bind_rx = state_mgr.take_bind_rx();
|
||||
|
||||
let mut xds_rx_for_task = xds_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
@ -116,7 +117,7 @@ pub async fn run(config: Arc<Config>) -> anyhow::Result<Bound> {
|
||||
mcp_state: mcp::App::new(stores.clone(), config.session_encoder.clone()),
|
||||
};
|
||||
|
||||
let gw = proxy::Gateway::new(Arc::new(pi), drain_rx.clone());
|
||||
let gw = proxy::Gateway::new(Arc::new(pi), drain_rx.clone(), bind_rx);
|
||||
|
||||
// Run the agentgateway in the data plane worker pool.
|
||||
let mut xds_rx_for_proxy = xds_rx.clone();
|
||||
|
||||
@ -21,7 +21,7 @@ use crate::types::discovery::{
|
||||
use crate::*;
|
||||
|
||||
fn run_test(req: &Request, routes: &[(&str, Vec<&str>, Vec<RouteMatch>)]) -> Option<String> {
|
||||
let stores = Stores::with_ipv6_enabled(true);
|
||||
let stores = Stores::default();
|
||||
let network = strng::literal!("network");
|
||||
let dummy_dest = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1000);
|
||||
|
||||
@ -1548,7 +1548,7 @@ fn bench(b: Bencher, (host, route): (u64, u64)) {
|
||||
.collect(),
|
||||
),
|
||||
});
|
||||
let stores = Stores::with_ipv6_enabled(true);
|
||||
let stores = Stores::default();
|
||||
let network = strng::literal!("network");
|
||||
let dummy_dest = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1000);
|
||||
let req = request("http://example.com", http::Method::GET, &[]);
|
||||
|
||||
@ -31,7 +31,7 @@ async fn setup_with_prefix(prefix: &str) -> (MockServer, Handler) {
|
||||
let parsed = reqwest::Url::parse(&host).unwrap();
|
||||
let config = crate::config::parse_config("{}".to_string(), None).unwrap();
|
||||
let encoder = config.session_encoder.clone();
|
||||
let stores = Stores::with_ipv6_enabled(config.ipv6_enabled);
|
||||
let stores = Stores::default();
|
||||
let client = Client::new(
|
||||
&client::Config {
|
||||
resolver_cfg: ResolverConfig::default(),
|
||||
|
||||
@ -17,7 +17,6 @@ use rand::RngExt;
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::{AbortHandle, JoinSet};
|
||||
use tokio_stream::StreamExt;
|
||||
use tracing::{Instrument, debug, error, event, info, info_span, warn};
|
||||
|
||||
use agent_core::strng;
|
||||
@ -127,84 +126,100 @@ impl TryFrom<&http::Uri> for HboneAddress {
|
||||
pub struct Gateway {
|
||||
pi: Arc<ProxyInputs>,
|
||||
drain: drain::DrainWatcher,
|
||||
bind_rx: tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>,
|
||||
}
|
||||
|
||||
impl Gateway {
|
||||
pub fn new(pi: Arc<ProxyInputs>, drain: DrainWatcher) -> Gateway {
|
||||
Gateway { drain, pi }
|
||||
pub fn new(
|
||||
pi: Arc<ProxyInputs>,
|
||||
drain: DrainWatcher,
|
||||
bind_rx: tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>,
|
||||
) -> Gateway {
|
||||
Gateway { drain, pi, bind_rx }
|
||||
}
|
||||
|
||||
pub async fn run(self) {
|
||||
pub async fn run(mut self) {
|
||||
let drain = self.drain.clone();
|
||||
let subdrain = self.drain.clone();
|
||||
let mut js = JoinSet::new();
|
||||
let (initial_binds, mut binds) = {
|
||||
let initial_binds = {
|
||||
let binds = self.pi.stores.read_binds();
|
||||
(binds.all(), binds.subscribe())
|
||||
binds.all()
|
||||
};
|
||||
let mut active: HashMap<SocketAddr, AbortHandle> = HashMap::new();
|
||||
let mut handle_bind = |js: &mut JoinSet<anyhow::Result<()>>, b: Event<Arc<Bind>>| {
|
||||
let b = match b {
|
||||
Event::Add(b) => b,
|
||||
Event::Remove(to_remove) => {
|
||||
if let Some(h) = active.remove(&to_remove.address) {
|
||||
h.abort();
|
||||
let mut handle_bind =
|
||||
|js: &mut JoinSet<anyhow::Result<()>>,
|
||||
b: Event<Arc<Bind>>,
|
||||
ack: Option<tokio::sync::oneshot::Sender<()>>| {
|
||||
let b = match b {
|
||||
Event::Add(b) => b,
|
||||
Event::Remove(to_remove) => {
|
||||
if let Some(h) = active.remove(&to_remove.address) {
|
||||
h.abort();
|
||||
}
|
||||
return;
|
||||
},
|
||||
};
|
||||
if active.contains_key(&b.address) {
|
||||
debug!("bind already exists");
|
||||
// Still ack — the bind address is already listening.
|
||||
if let Some(tx) = ack {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
return;
|
||||
},
|
||||
};
|
||||
if active.contains_key(&b.address) {
|
||||
debug!("bind already exists");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
debug!("add bind {}", b.address);
|
||||
if self.pi.cfg.threading_mode == crate::ThreadingMode::ThreadPerCore {
|
||||
let core_ids = core_affinity::get_core_ids().unwrap();
|
||||
let _ = core_ids
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
let subdrain = subdrain.clone();
|
||||
let pi = self.pi.clone();
|
||||
let b = b.clone();
|
||||
std::thread::spawn(move || {
|
||||
let res = core_affinity::set_for_current(id);
|
||||
if !res {
|
||||
panic!("failed to set current CPU")
|
||||
}
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
let _ = Self::run_bind(pi.clone(), subdrain.clone(), b.clone())
|
||||
.in_current_span()
|
||||
.await;
|
||||
})
|
||||
debug!("add bind {}", b.address);
|
||||
if self.pi.cfg.threading_mode == crate::ThreadingMode::ThreadPerCore {
|
||||
let core_ids = core_affinity::get_core_ids().unwrap();
|
||||
let _ = core_ids
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
let subdrain = subdrain.clone();
|
||||
let pi = self.pi.clone();
|
||||
let b = b.clone();
|
||||
std::thread::spawn(move || {
|
||||
let res = core_affinity::set_for_current(id);
|
||||
if !res {
|
||||
panic!("failed to set current CPU")
|
||||
}
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
let _ =
|
||||
Self::run_bind(pi.clone(), subdrain.clone(), b.clone(), None)
|
||||
.in_current_span()
|
||||
.await;
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
} else {
|
||||
let task =
|
||||
js.spawn(Self::run_bind(self.pi.clone(), subdrain.clone(), b.clone()).in_current_span());
|
||||
active.insert(b.address, task);
|
||||
}
|
||||
};
|
||||
.collect::<Vec<_>>();
|
||||
// ThreadPerCore spawns OS threads — we can't easily plumb the ack
|
||||
// through all of them. Send the ack now; in practice ThreadPerCore
|
||||
// binds synchronously before spawning the accept loop.
|
||||
if let Some(tx) = ack {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
} else {
|
||||
let task = js.spawn(
|
||||
Self::run_bind(self.pi.clone(), subdrain.clone(), b.clone(), ack)
|
||||
.in_current_span(),
|
||||
);
|
||||
active.insert(b.address, task);
|
||||
}
|
||||
};
|
||||
for bind in initial_binds {
|
||||
handle_bind(&mut js, Event::Add(bind))
|
||||
handle_bind(&mut js, Event::Add(bind), None)
|
||||
}
|
||||
|
||||
let wait = drain.wait_for_drain();
|
||||
tokio::pin!(wait);
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(res) = binds.next() => {
|
||||
let Ok(res) = res else {
|
||||
// TODO: move to unbuffered
|
||||
warn!("lagged on bind update");
|
||||
continue;
|
||||
};
|
||||
handle_bind(&mut js, res);
|
||||
Some(bind_event) = self.bind_rx.recv() => {
|
||||
handle_bind(&mut js, bind_event.event, bind_event.ack);
|
||||
}
|
||||
Some(res) = js.join_next() => {
|
||||
warn!("bind complete {res:?}");
|
||||
@ -225,6 +240,7 @@ impl Gateway {
|
||||
pi: Arc<ProxyInputs>,
|
||||
drain: DrainWatcher,
|
||||
b: Arc<Bind>,
|
||||
ack: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
) -> anyhow::Result<()> {
|
||||
let min_deadline = pi.cfg.termination_min_deadline;
|
||||
let max_deadline = pi.cfg.termination_max_deadline;
|
||||
@ -257,6 +273,10 @@ impl Gateway {
|
||||
} else {
|
||||
(pi, TcpListener::bind(b.address).await?)
|
||||
};
|
||||
// Listener is bound — notify the caller that it's safe to send traffic.
|
||||
if let Some(tx) = ack {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
info!(bind = name.as_str(), "started bind");
|
||||
let component = format!("bind {name}");
|
||||
|
||||
|
||||
@ -19,6 +19,9 @@ pub struct StateManager {
|
||||
|
||||
#[serde(skip_serializing)]
|
||||
xds_client: Option<agent_xds::AdsClient>,
|
||||
|
||||
#[serde(skip_serializing)]
|
||||
bind_rx: Option<tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>>,
|
||||
}
|
||||
|
||||
pub const ADDRESS_TYPE: Strng = strng::literal!("type.googleapis.com/istio.workload.Address");
|
||||
@ -35,7 +38,8 @@ impl StateManager {
|
||||
awaiting_ready: tokio::sync::watch::Sender<()>,
|
||||
) -> anyhow::Result<Self> {
|
||||
let xds = &config.xds;
|
||||
let stores = Stores::with_ipv6_enabled(config.ipv6_enabled);
|
||||
let crate::store::StoresWithBindRx { stores, bind_rx } =
|
||||
Stores::with_ipv6_enabled(config.ipv6_enabled);
|
||||
let xds_client = if let Some(addr) = &xds.address {
|
||||
let connector = control::grpc_connector(
|
||||
client.clone(),
|
||||
@ -72,13 +76,27 @@ impl StateManager {
|
||||
};
|
||||
Box::pin(local_client.run()).await?;
|
||||
}
|
||||
Ok(Self { stores, xds_client })
|
||||
Ok(Self {
|
||||
stores,
|
||||
xds_client,
|
||||
bind_rx: Some(bind_rx),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn stores(&self) -> Stores {
|
||||
self.stores.clone()
|
||||
}
|
||||
|
||||
/// Take the bind event receiver. Must be called exactly once before
|
||||
/// passing it to [`crate::proxy::gateway::Gateway::new`].
|
||||
pub fn take_bind_rx(
|
||||
&mut self,
|
||||
) -> tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent> {
|
||||
self.bind_rx
|
||||
.take()
|
||||
.expect("bind_rx already taken or StateManager not properly initialized")
|
||||
}
|
||||
|
||||
pub async fn run(self) -> anyhow::Result<()> {
|
||||
match self.xds_client {
|
||||
Some(xds) => xds.run().await.map_err(|e| anyhow::anyhow!(e)),
|
||||
|
||||
@ -2,9 +2,7 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use agent_xds::{RejectedConfig, XdsUpdate};
|
||||
use futures_core::Stream;
|
||||
use itertools::Itertools;
|
||||
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
|
||||
use tracing::{Level, instrument};
|
||||
|
||||
use crate::cel::ContextBuilder;
|
||||
@ -62,7 +60,7 @@ pub struct Store {
|
||||
service_routes: HashMap<NamespacedHostname, RouteSet>,
|
||||
service_tcp_routes: HashMap<NamespacedHostname, TCPRouteSet>,
|
||||
|
||||
tx: tokio::sync::broadcast::Sender<Event<Arc<Bind>>>,
|
||||
tx: tokio::sync::mpsc::UnboundedSender<crate::store::BindEvent>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone)]
|
||||
@ -369,9 +367,11 @@ pub struct LLMResponsePolicies {
|
||||
pub prompt_guard: Vec<ResponseGuard>,
|
||||
}
|
||||
|
||||
impl Default for Store {
|
||||
fn default() -> Self {
|
||||
Self::with_ipv6_enabled(true)
|
||||
impl Store {
|
||||
/// Create a store with a disconnected channel (for tests that don't need bind events).
|
||||
pub fn new_disconnected(ipv6_enabled: bool) -> Self {
|
||||
let (store, _rx) = Self::with_ipv6_enabled(ipv6_enabled);
|
||||
store
|
||||
}
|
||||
}
|
||||
|
||||
@ -383,9 +383,11 @@ pub struct RoutePath<'a> {
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn with_ipv6_enabled(ipv6_enabled: bool) -> Self {
|
||||
let (tx, _) = tokio::sync::broadcast::channel(1000);
|
||||
Self {
|
||||
pub fn with_ipv6_enabled(
|
||||
ipv6_enabled: bool,
|
||||
) -> (Self, tokio::sync::mpsc::UnboundedReceiver<crate::store::BindEvent>) {
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let store = Self {
|
||||
ipv6_enabled,
|
||||
binds: Default::default(),
|
||||
resources: Default::default(),
|
||||
@ -398,13 +400,8 @@ impl Store {
|
||||
service_routes: Default::default(),
|
||||
service_tcp_routes: Default::default(),
|
||||
tx,
|
||||
}
|
||||
}
|
||||
pub fn subscribe(
|
||||
&self,
|
||||
) -> impl Stream<Item = Result<Event<Arc<Bind>>, BroadcastStreamRecvError>> + use<> {
|
||||
let sub = self.tx.subscribe();
|
||||
tokio_stream::wrappers::BroadcastStream::new(sub)
|
||||
};
|
||||
(store, rx)
|
||||
}
|
||||
|
||||
pub fn route_policies(&self, path: &RoutePath<'_>, inline: &[TrafficPolicy]) -> RoutePolicies {
|
||||
@ -811,7 +808,10 @@ impl Store {
|
||||
)]
|
||||
pub fn remove_bind(&mut self, bind: BindKey) {
|
||||
if let Some(old) = self.binds.remove(&bind) {
|
||||
let _ = self.tx.send(Event::Remove(old));
|
||||
let _ = self.tx.send(crate::store::BindEvent {
|
||||
event: Event::Remove(old),
|
||||
ack: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
#[instrument(
|
||||
@ -911,7 +911,24 @@ impl Store {
|
||||
skip_all,
|
||||
fields(bind=%bind.key),
|
||||
)]
|
||||
pub fn insert_bind(&mut self, mut bind: Bind) {
|
||||
pub fn insert_bind(&mut self, bind: Bind) {
|
||||
self.insert_bind_with_ack(bind, None);
|
||||
}
|
||||
|
||||
/// Insert a bind and return a [`oneshot::Receiver`] that resolves once the
|
||||
/// TCP listener is actually bound. Use this when you need to block until
|
||||
/// the listener is ready (e.g. before telling a sandbox it can send traffic).
|
||||
#[instrument(
|
||||
level = Level::INFO,
|
||||
name="insert_bind_with_ack",
|
||||
skip_all,
|
||||
fields(bind=%bind.key),
|
||||
)]
|
||||
pub fn insert_bind_with_ack(
|
||||
&mut self,
|
||||
mut bind: Bind,
|
||||
ack: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
) {
|
||||
debug!(bind=%bind.key, "insert bind");
|
||||
|
||||
// Insert any staged listeners
|
||||
@ -934,8 +951,10 @@ impl Store {
|
||||
}
|
||||
let arc = Arc::new(bind);
|
||||
self.binds.insert(arc.key.clone(), arc.clone());
|
||||
// ok to have no subs
|
||||
let _ = self.tx.send(Event::Add(arc));
|
||||
let _ = self.tx.send(crate::store::BindEvent {
|
||||
event: Event::Add(arc),
|
||||
ack,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn insert_backend(&mut self, key: BackendKey, b: BackendWithPolicies) {
|
||||
@ -1508,7 +1527,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn route_policies_are_kind_scoped() {
|
||||
let mut store = Store::default();
|
||||
let mut store = Store::new_disconnected(true);
|
||||
let listener = listener();
|
||||
|
||||
let http_route = route("r", "ns", Some("HTTPRoute"));
|
||||
@ -1539,7 +1558,7 @@ mod tests {
|
||||
/// Tests that frontend policies at listener level take precedence over gateway level policies
|
||||
#[test]
|
||||
fn frontend_policy_listener_precedence() {
|
||||
let mut store = Store::default();
|
||||
let mut store = Store::new_disconnected(true);
|
||||
let listener = listener();
|
||||
|
||||
// Insert both gateway and listener level frontend policies
|
||||
@ -1566,7 +1585,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn frontend_network_authorization_policies_merge() {
|
||||
let mut store = Store::default();
|
||||
let mut store = Store::new_disconnected(true);
|
||||
let listener = listener();
|
||||
insert_gateway_level_network_authorization_policy(
|
||||
&mut store,
|
||||
|
||||
@ -24,26 +24,52 @@ pub enum Event<T> {
|
||||
Remove(T),
|
||||
}
|
||||
|
||||
/// A bind event with an optional acknowledgment channel.
|
||||
///
|
||||
/// When `ack` is `Some`, the consumer should send `()` on it after the TCP
|
||||
/// listener is actually bound. This lets callers of `insert_bind` wait for
|
||||
/// the listener to be ready instead of racing.
|
||||
pub struct BindEvent {
|
||||
pub event: Event<Arc<crate::types::agent::Bind>>,
|
||||
pub ack: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BindEvent {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("BindEvent")
|
||||
.field("event", &self.event)
|
||||
.field("has_ack", &self.ack.is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Stores {
|
||||
pub discovery: discovery::StoreUpdater,
|
||||
pub binds: binds::StoreUpdater,
|
||||
}
|
||||
|
||||
/// Return type from [`Stores::with_ipv6_enabled`] — the stores plus the bind
|
||||
/// event receiver that [`crate::proxy::gateway::Gateway`] must consume.
|
||||
pub struct StoresWithBindRx {
|
||||
pub stores: Stores,
|
||||
pub bind_rx: tokio::sync::mpsc::UnboundedReceiver<BindEvent>,
|
||||
}
|
||||
|
||||
impl Default for Stores {
|
||||
fn default() -> Self {
|
||||
Self::with_ipv6_enabled(true)
|
||||
Self::with_ipv6_enabled(true).stores
|
||||
}
|
||||
}
|
||||
|
||||
impl Stores {
|
||||
pub fn with_ipv6_enabled(ipv6_enabled: bool) -> Stores {
|
||||
Stores {
|
||||
pub fn with_ipv6_enabled(ipv6_enabled: bool) -> StoresWithBindRx {
|
||||
let (bind_store, bind_rx) = binds::Store::with_ipv6_enabled(ipv6_enabled);
|
||||
let stores = Stores {
|
||||
discovery: discovery::StoreUpdater::new(Arc::new(RwLock::new(discovery::Store::new()))),
|
||||
binds: binds::StoreUpdater::new(Arc::new(RwLock::new(binds::Store::with_ipv6_enabled(
|
||||
ipv6_enabled,
|
||||
)))),
|
||||
}
|
||||
binds: binds::StoreUpdater::new(Arc::new(RwLock::new(bind_store))),
|
||||
};
|
||||
StoresWithBindRx { stores, bind_rx }
|
||||
}
|
||||
pub fn read_binds(&self) -> std::sync::RwLockReadGuard<'_, store::BindStore> {
|
||||
self.binds.read()
|
||||
|
||||
@ -735,7 +735,8 @@ pub fn setup_proxy_test(cfg: &str) -> anyhow::Result<TestBind> {
|
||||
agent_core::telemetry::testing::setup_test_logging();
|
||||
let config = crate::config::parse_config(cfg.to_string(), None)?;
|
||||
let encoder = config.session_encoder.clone();
|
||||
let stores = Stores::with_ipv6_enabled(config.ipv6_enabled);
|
||||
let crate::store::StoresWithBindRx { stores, bind_rx: _bind_rx } =
|
||||
Stores::with_ipv6_enabled(config.ipv6_enabled);
|
||||
let client = client::Client::new(&config.dns, None, Default::default(), None);
|
||||
let (drain_tx, drain_rx) = drain::new();
|
||||
let pi = Arc::new(ProxyInputs {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user