diff --git a/.github/buildomat/common.sh b/.github/buildomat/common.sh index 965e5b78..34cc04de 100644 --- a/.github/buildomat/common.sh +++ b/.github/buildomat/common.sh @@ -7,7 +7,7 @@ # track when we exceed the current ceiling. The underlying intention is to grow # deliberately and thoughtfully, given the limited space on the ASIC. if [ "$MULTICAST" == "" ]; then - TOFINO_STAGES=17 + TOFINO_STAGES=18 else TOFINO_STAGES=19 fi diff --git a/Cargo.lock b/Cargo.lock index f7272829..7da07251 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1741,6 +1741,7 @@ dependencies = [ "dpd-types-versions", "dropshot 0.18.0", "dropshot-api-manager-types", + "uuid", ] [[package]] @@ -7401,6 +7402,7 @@ dependencies = [ "tabwriter", "thiserror 2.0.18", "tokio", + "uuid", ] [[package]] diff --git a/aal/src/match_action.rs b/aal/src/match_action.rs index 5a776e15..3001463a 100644 --- a/aal/src/match_action.rs +++ b/aal/src/match_action.rs @@ -410,6 +410,8 @@ impl TryFrom<&ValueTypes> for u8 { } } +unwrap_value_entry!(u8); + impl From for ValueTypes { fn from(v: u16) -> ValueTypes { ValueTypes::U64(v as u64) diff --git a/asic/src/softnpu/table.rs b/asic/src/softnpu/table.rs index 56d9c1fe..5e04aee2 100644 --- a/asic/src/softnpu/table.rs +++ b/asic/src/softnpu/table.rs @@ -119,8 +119,16 @@ impl TableOps for Table { action_data.action.as_str(), ) { // TODO: implement mappings for natv6 actions - (TableType::PortAddrIpv4, "claimv4") => ("local", Vec::new()), - (TableType::PortAddrIpv6, "claimv6") => ("local", Vec::new()), + // sidecar-lite's local action carries the routing table id; + // IPv4 switch addresses belong to the default table. + (TableType::PortAddrIpv4, "claimv4") => ("local", vec![0]), + (TableType::PortAddrIpv6, "claimv6") => { + let table_id = u8::try_from( + &action_data.arg_by_name("routing_table_id")?.value, + ) + .map_err(|e| AsicError::InvalidArg(e.into()))?; + ("local", vec![table_id]) + } (TableType::RouteIdxIpv4, "index") => { let mut params = Vec::new(); for arg in action_data.args.iter() { @@ -632,10 +640,10 @@ fn keyset_data(match_data: Vec, table: TableType) -> Vec { serialize_value_type(&x, &mut data); keyset_data.extend_from_slice(&data[..2]); } - TableType::RouteIdxIpv4 => { - // "idx" => exact => bit<16> + TableType::RouteIdxIpv4 | TableType::RouteIdxIpv6 => { + // "routing_table_id" => exact => bit<8> serialize_value_type(&x, &mut data); - keyset_data.extend_from_slice(&data[..2]); + keyset_data.push(data[0]); } TableType::NatIngressIpv4 => { // "dst_addr" => hdr.ipv4.dst: exact => bit<32> @@ -725,3 +733,48 @@ fn serialize_value_type_be(x: &ValueTypes, data: &mut Vec) { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn route_key( + table_id: u8, + prefix: ValueTypes, + len: u16, + ) -> Vec { + vec![ + MatchEntryField { + name: "routing_table_id".into(), + value: MatchEntryValue::Value(table_id.into()), + }, + MatchEntryField { + name: "dst_addr".into(), + value: MatchEntryValue::Lpm(aal::MatchLpm { prefix, len }), + }, + ] + } + + #[test] + fn route_index_serializes_composite_key() { + assert_eq!( + keyset_data( + route_key(2, ValueTypes::U64(0xc000_0200), 24), + TableType::RouteIdxIpv4, + ), + [2, 192, 0, 2, 0, 24] + ); + + let prefix = 0x20010db8000000000000000000000000u128.to_be_bytes(); + let mut expected = vec![3]; + expected.extend_from_slice(&prefix); + expected.push(32); + assert_eq!( + keyset_data( + route_key(3, ValueTypes::Ptr(prefix.to_vec()), 32), + TableType::RouteIdxIpv6, + ), + expected + ); + } +} diff --git a/dpd-api/Cargo.toml b/dpd-api/Cargo.toml index 10730af0..20b17465 100644 --- a/dpd-api/Cargo.toml +++ b/dpd-api/Cargo.toml @@ -7,3 +7,4 @@ edition = "2024" dpd-types-versions.workspace = true dropshot.workspace = true dropshot-api-manager-types.workspace = true +uuid.workspace = true diff --git a/dpd-api/src/lib.rs b/dpd-api/src/lib.rs index 7441178e..657c2fe5 100644 --- a/dpd-api/src/lib.rs +++ b/dpd-api/src/lib.rs @@ -9,13 +9,14 @@ use std::collections::BTreeMap; use std::net::{Ipv4Addr, Ipv6Addr}; -use dpd_types_versions::{latest, v1, v4, v7}; +use dpd_types_versions::{latest, v1, v4, v6, v7}; use dropshot::{ EmptyScanParams, HttpError, HttpResponseCreated, HttpResponseDeleted, HttpResponseOk, HttpResponseUpdatedNoContent, PaginationParams, Path, Query, RequestContext, ResultsPage, TypedBody, }; use dropshot_api_manager_types::api_versions; +use uuid::Uuid; api_versions!([ // WHEN CHANGING THE API (part 1 of 2): @@ -29,6 +30,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (14, MODULAR_ROUTER), (13, ALLOW_DDM_TRAFFIC), (12, PRBS_ERROR_TRACKING), (11, WALLCLOCK_HISTORY), @@ -182,8 +184,9 @@ pub trait DpdApi { ) -> Result; /** - * Fetch the configured IPv6 routes, mapping IPv6 CIDR blocks to the switch port - * used for sending out that traffic, and optionally a gateway. + * Fetch the IPv6 routes configured on the default routing table, mapping + * IPv6 CIDR blocks to the switch port used for sending out that traffic, + * and optionally a gateway. */ #[endpoint { method = GET, @@ -197,7 +200,8 @@ pub trait DpdApi { ) -> Result>, HttpError>; /** - * Get a single IPv6 route, by its IPv6 CIDR block. + * Get a single IPv6 route on the default routing table, by its IPv6 CIDR + * block. */ #[endpoint { method = GET, @@ -205,11 +209,12 @@ pub trait DpdApi { }] async fn route_ipv6_get( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result>, HttpError>; /** - * Route an IPv6 subnet to a link and a nexthop gateway. + * Route an IPv6 subnet to a link and a nexthop gateway on the default + * routing table. * * This call can be used to create a new single-path route or to add new targets * to a multipath route. @@ -224,7 +229,8 @@ pub trait DpdApi { ) -> Result; /** - * Route an IPv6 subnet to a link and a nexthop gateway. + * Route an IPv6 subnet to a link and a nexthop gateway on the default + * routing table. * * This call can be used to create a new single-path route or to replace any * existing routes with a new single-path route. @@ -239,7 +245,8 @@ pub trait DpdApi { ) -> Result; /** - * Remove an IPv6 route, by its IPv6 CIDR block. + * Remove an IPv6 route on the default routing table, by its IPv6 CIDR + * block. */ #[endpoint { method = DELETE, @@ -247,11 +254,12 @@ pub trait DpdApi { }] async fn route_ipv6_delete( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result; /** - * Remove a single target for the given IPv6 subnet + * Remove a single target for the given IPv6 subnet on the default routing + * table */ #[endpoint { method = DELETE, @@ -259,12 +267,13 @@ pub trait DpdApi { }] async fn route_ipv6_delete_target( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result; /** - * Fetch the configured IPv4 routes, mapping IPv4 CIDR blocks to the switch port - * used for sending out that traffic, and optionally a gateway. + * Fetch the IPv4 routes configured on the default routing table, mapping + * IPv4 CIDR blocks to the switch port used for sending out that traffic, + * and optionally a gateway. */ #[endpoint { method = GET, @@ -302,7 +311,8 @@ pub trait DpdApi { } /** - * Get the configured route for the given IPv4 subnet. + * Get the configured route for the given IPv4 subnet on the default routing + * table. */ #[endpoint { method = GET, @@ -311,7 +321,7 @@ pub trait DpdApi { }] async fn route_ipv4_get( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result>, HttpError>; #[endpoint { @@ -336,7 +346,8 @@ pub trait DpdApi { } /** - * Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6). + * Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on + * the default routing table. * * This call can be used to create a new single-path route or to add new targets * to a multipath route. @@ -390,7 +401,8 @@ pub trait DpdApi { } /** - * Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6). + * Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on + * the default routing table. * * This call can be used to create a new single-path route or to replace any * existing routes with a new single-path route. @@ -444,7 +456,7 @@ pub trait DpdApi { } /** - * Remove all targets for the given subnet + * Remove all targets for the given subnet on the default routing table */ #[endpoint { method = DELETE, @@ -452,11 +464,12 @@ pub trait DpdApi { }] async fn route_ipv4_delete( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result; /** - * Remove a single target for the given IPv4 subnet (IPv4 or IPv6 next hop) + * Remove a single target for the given IPv4 subnet on the default routing + * table (IPv4 or IPv6 next hop) */ #[endpoint { method = DELETE, @@ -465,7 +478,7 @@ pub trait DpdApi { }] async fn route_ipv4_delete_target( rqctx: RequestContext, - path: Path, + path: Path, ) -> Result; /** @@ -484,6 +497,248 @@ pub trait DpdApi { Self::route_ipv4_delete_target(rqctx, path.map(Into::into)).await } + /** + * List the routers with their endpoints. + */ + #[endpoint { + method = GET, + path = "/router", + versions = VERSION_MODULAR_ROUTER.. + }] + async fn router_list( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /** + * Get a router's endpoint. + */ + #[endpoint { + method = GET, + path = "/router/{router_id}", + versions = VERSION_MODULAR_ROUTER.. + }] + async fn router_get( + rqctx: RequestContext, + path: Path, + ) -> Result, HttpError>; + + /** + * Create a router with the given endpoint. An address can be the + * endpoint of only one router, and a router's endpoint can't be changed. + * The default routing table's uuid can't be used. + */ + #[endpoint { + method = PUT, + path = "/router/{router_id}", + versions = VERSION_MODULAR_ROUTER.. + }] + async fn router_create( + rqctx: RequestContext, + path: Path, + endpoint: TypedBody, + ) -> Result; + + /** + * Delete a router, along with its endpoint and routes. The default + * routing table's uuid can't be used. + */ + #[endpoint { + method = DELETE, + path = "/router/{router_id}", + versions = VERSION_MODULAR_ROUTER.. + }] + async fn router_delete( + rqctx: RequestContext, + path: Path, + ) -> Result; + + /** + * Fetch the IPv6 routes configured on the given router, mapping IPv6 + * CIDR blocks to the switch port used for sending out that traffic, and + * optionally a gateway. + */ + #[endpoint { + method = GET, + path = "/router/{router_id}/route/ipv6", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv6_list( + rqctx: RequestContext, + path: Path, + query: Query< + PaginationParams, + >, + ) -> Result>, HttpError>; + + /** + * Get a single IPv6 route on the given router, by its IPv6 CIDR block. + */ + #[endpoint { + method = GET, + path = "/router/{router_id}/route/ipv6/{cidr}", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv6_get( + rqctx: RequestContext, + path: Path, + ) -> Result>, HttpError>; + + /** + * Route an IPv6 subnet to a link and a nexthop gateway on the given + * router. + * + * This call can be used to create a new single-path route or to add new + * targets to a multipath route. + */ + #[endpoint { + method = POST, + path = "/router/{router_id}/route/ipv6", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv6_add( + rqctx: RequestContext, + path: Path, + update: TypedBody, + ) -> Result; + + /** + * Route an IPv6 subnet to a link and a nexthop gateway on the given + * router. + * + * This call can be used to create a new single-path route or to replace + * any existing routes with a new single-path route. + */ + #[endpoint { + method = PUT, + path = "/router/{router_id}/route/ipv6", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv6_set( + rqctx: RequestContext, + path: Path, + update: TypedBody, + ) -> Result; + + /** + * Remove an IPv6 route on the given router, by its IPv6 CIDR block. + */ + #[endpoint { + method = DELETE, + path = "/router/{router_id}/route/ipv6/{cidr}", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv6_delete( + rqctx: RequestContext, + path: Path, + ) -> Result; + + /** + * Remove a single target for the given IPv6 subnet on the given router. + */ + #[endpoint { + method = DELETE, + path = "/router/{router_id}/route/ipv6/{cidr}/{port_id}/{link_id}/{tgt_ip}", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv6_delete_target( + rqctx: RequestContext, + path: Path, + ) -> Result; + + /** + * Fetch the IPv4 routes configured on the given router, mapping IPv4 + * CIDR blocks to the switch port used for sending out that traffic, and + * optionally a gateway. + */ + #[endpoint { + method = GET, + path = "/router/{router_id}/route/ipv4", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv4_list( + rqctx: RequestContext, + path: Path, + query: Query< + PaginationParams, + >, + ) -> Result>, HttpError>; + + /** + * Get the configured route for the given IPv4 subnet on the given router. + */ + #[endpoint { + method = GET, + path = "/router/{router_id}/route/ipv4/{cidr}", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv4_get( + rqctx: RequestContext, + path: Path, + ) -> Result>, HttpError>; + + /** + * Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on + * the given router. + * + * This call can be used to create a new single-path route or to add new + * targets to a multipath route. + */ + #[endpoint { + method = POST, + path = "/router/{router_id}/route/ipv4", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv4_add( + rqctx: RequestContext, + path: Path, + update: TypedBody, + ) -> Result; + + /** + * Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on + * the given router. + * + * This call can be used to create a new single-path route or to replace + * any existing routes with a new single-path route. + */ + #[endpoint { + method = PUT, + path = "/router/{router_id}/route/ipv4", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv4_set( + rqctx: RequestContext, + path: Path, + update: TypedBody, + ) -> Result; + + /** + * Remove all targets for the given subnet on the given router. + */ + #[endpoint { + method = DELETE, + path = "/router/{router_id}/route/ipv4/{cidr}", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv4_delete( + rqctx: RequestContext, + path: Path, + ) -> Result; + + /** + * Remove a single target for the given IPv4 subnet on the given router + * (IPv4 or IPv6 next hop). + */ + #[endpoint { + method = DELETE, + path = "/router/{router_id}/route/ipv4/{cidr}/{port_id}/{link_id}/{tgt_ip}", + versions = VERSION_MODULAR_ROUTER.., + }] + async fn router_route_ipv4_delete_target( + rqctx: RequestContext, + path: Path, + ) -> Result; + /// List all switch ports on the system. #[endpoint { method = GET, diff --git a/dpd-client/tests/integration_tests/common.rs b/dpd-client/tests/integration_tests/common.rs index 0fcebdc8..9e77f71f 100644 --- a/dpd-client/tests/integration_tests/common.rs +++ b/dpd-client/tests/integration_tests/common.rs @@ -23,6 +23,7 @@ use dpd_client::Client; use dpd_client::ClientInfo; use dpd_client::ClientState; use dpd_client::types; +pub use dpd_types::route::DEFAULT_ROUTER; use packet::Endpoint; use packet::Packet; use packet::arp; @@ -1514,6 +1515,7 @@ pub fn tofino_flow_hash( pub mod prelude { pub use super::ADMIN_LOCAL_MULTICAST_PREFIX; + pub use super::DEFAULT_ROUTER; pub use super::L4Protocol; pub use super::NO_PORT; pub use super::PhysPort; diff --git a/dpd-client/tests/integration_tests/mod.rs b/dpd-client/tests/integration_tests/mod.rs index 29c5f2e6..4899654f 100644 --- a/dpd-client/tests/integration_tests/mod.rs +++ b/dpd-client/tests/integration_tests/mod.rs @@ -16,5 +16,6 @@ mod nat; mod port_api; mod route_ipv4; mod route_ipv6; +mod router; mod service; mod table_tests; diff --git a/dpd-client/tests/integration_tests/router.rs b/dpd-client/tests/integration_tests/router.rs new file mode 100644 index 00000000..8d5aaf59 --- /dev/null +++ b/dpd-client/tests/integration_tests/router.rs @@ -0,0 +1,531 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use std::collections::HashMap; +use std::net::Ipv6Addr; +use std::sync::Arc; + +use dpd_client::ClientInfo; +use dpd_client::types; +use oxnet::Ipv4Net; +use packet::Endpoint; +use packet::Packet; +use packet::eth; +use packet::geneve; +use reqwest::StatusCode; +use uuid::Uuid; + +use crate::integration_tests::common; +use crate::integration_tests::common::prelude::*; + +fn ipv4_route(switch: &Switch, port: u16, gw: &str) -> types::Ipv4Route { + let (port_id, link_id) = switch.link_id(PhysPort(port)).unwrap(); + types::Ipv4Route { + port_id, + link_id, + tgt_ip: gw.parse().unwrap(), + tag: "testing".into(), + vlan_id: None, + } +} + +fn route_update( + cidr: Ipv4Net, + target: &types::Ipv4Route, +) -> types::Ipv4RouteUpdate { + types::Ipv4RouteUpdate { + cidr, + target: types::RouteTarget::V4(target.clone()), + replace: false, + } +} + +fn ipv6(addr: &str) -> Ipv6Addr { + addr.parse().unwrap() +} + +/// The same prefix can be routed differently by different routers. +#[tokio::test] +#[ignore] +async fn test_same_prefix_two_routers() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let router = Uuid::new_v4(); + let cidr: Ipv4Net = "10.201.0.0/24".parse().unwrap(); + let default_target = ipv4_route(switch, 10, "10.10.10.1"); + let router_target = ipv4_route(switch, 14, "10.10.14.1"); + + client.router_create(&router, &ipv6("fd00:201::1")).await?; + client.route_ipv4_add(&route_update(cidr, &default_target)).await?; + client + .router_route_ipv4_add(&router, &route_update(cidr, &router_target)) + .await?; + + let found = client.route_ipv4_get(&cidr).await?; + assert_eq!(*found, vec![types::Route::V4(default_target)]); + let found = client.router_route_ipv4_get(&router, &cidr).await?; + assert_eq!(*found, vec![types::Route::V4(router_target)]); + + client.route_ipv4_delete(&cidr).await?; + client.router_delete(&router).await?; + Ok(()) +} + +/// Deleting a router removes its routes and endpoint and doesn't affect +/// another router. +#[tokio::test] +#[ignore] +async fn test_router_delete() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let router = Uuid::new_v4(); + let cidr: Ipv4Net = "10.202.0.0/24".parse().unwrap(); + let target = ipv4_route(switch, 10, "10.10.10.1"); + let endpoint = ipv6("fd00:202::2"); + + client.router_create(&router, &endpoint).await?; + client.route_ipv4_add(&route_update(cidr, &target)).await?; + client.router_route_ipv4_add(&router, &route_update(cidr, &target)).await?; + + client.router_delete(&router).await?; + + assert!(!client.router_list().await?.contains_key(&router.to_string())); + let err = client.router_route_ipv4_get(&router, &cidr).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::NOT_FOUND)); + assert_eq!(client.route_ipv4_get(&cidr).await?.len(), 1); + + // The deleted router's endpoint is free to be claimed again. + let again = Uuid::new_v4(); + client.router_create(&again, &endpoint).await?; + + client.route_ipv4_delete(&cidr).await?; + client.router_delete(&again).await?; + Ok(()) +} + +/// Operations naming a router that was never created fail with 404, except +/// delete. +#[tokio::test] +#[ignore] +async fn test_uncreated_router() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let router = Uuid::new_v4(); + let cidr: Ipv4Net = "10.203.0.0/24".parse().unwrap(); + let target = ipv4_route(switch, 10, "10.10.10.1"); + + let statuses = [ + client.router_route_ipv4_list(&router, None, None).await.map(|_| ()), + client.router_route_ipv4_get(&router, &cidr).await.map(|_| ()), + client + .router_route_ipv4_add(&router, &route_update(cidr, &target)) + .await + .map(|_| ()), + client.router_route_ipv4_delete(&router, &cidr).await.map(|_| ()), + client.router_route_ipv6_list(&router, None, None).await.map(|_| ()), + client.router_get(&router).await.map(|_| ()), + ] + .map(|r| r.unwrap_err().status()); + for status in statuses { + assert_eq!(status, Some(StatusCode::NOT_FOUND)); + } + let resp = client.router_delete(&router).await?; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + Ok(()) +} + +/// An address can be the endpoint of only one router, and an endpoint can't +/// take a loopback's address. +#[tokio::test] +#[ignore] +async fn test_endpoint_unique() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let a = Uuid::new_v4(); + let b = Uuid::new_v4(); + let endpoint = ipv6("fd00:204::1"); + let lo = types::Ipv6Entry { + tag: "testing".into(), + addr: "fd00:204::9".parse().unwrap(), + }; + + client.router_create(&a, &endpoint).await?; + let err = client.router_create(&b, &endpoint).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::CONFLICT)); + client.loopback_ipv6_create(&lo).await?; + let err = client.router_create(&b, &lo.addr).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::CONFLICT)); + + // None of the refused requests left anything behind. + assert_eq!( + client.router_list().await?.into_inner(), + HashMap::from([(a.to_string(), endpoint)]) + ); + + client.loopback_ipv6_delete(&lo.addr).await?; + client.router_delete(&a).await?; + Ok(()) +} + +/// A router's endpoint is listed as a loopback, tagged with the router's +/// uuid. It can't be added or deleted through the loopback API. +#[tokio::test] +#[ignore] +async fn test_endpoint_in_loopback_list() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let router = Uuid::new_v4(); + let endpoint = ipv6("fd00:20a::1"); + let tag_of = |loopbacks: &[types::Ipv6Entry]| { + loopbacks.iter().find(|e| e.addr == endpoint).map(|e| e.tag.clone()) + }; + + client.router_create(&router, &endpoint).await?; + let tag = tag_of(&client.loopback_ipv6_list().await?); + assert_eq!(tag, Some(router.to_string())); + + let err = client.loopback_ipv6_delete(&endpoint).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::CONFLICT)); + + let lo = types::Ipv6Entry { tag: "testing".into(), addr: endpoint }; + let err = client.loopback_ipv6_create(&lo).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::CONFLICT)); + let tag = tag_of(&client.loopback_ipv6_list().await?); + assert_eq!(tag, Some(router.to_string())); + + client.router_delete(&router).await?; + assert_eq!(tag_of(&client.loopback_ipv6_list().await?), None); + Ok(()) +} + +/// Creating a router again with the same endpoint succeeds, and with a +/// different endpoint fails. +#[tokio::test] +#[ignore] +async fn test_router_create_delete() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let router = Uuid::new_v4(); + let endpoint = ipv6("fd00:205::1"); + + for _ in 0..2 { + let resp = client.router_create(&router, &endpoint).await?; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + } + let err = + client.router_create(&router, &ipv6("fd00:205::2")).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::CONFLICT)); + assert_eq!(*client.router_get(&router).await?, endpoint); + assert!(client.router_list().await?.contains_key(&router.to_string())); + + let resp = client.router_delete(&router).await?; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + let err = client.router_get(&router).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::NOT_FOUND)); + Ok(()) +} + +/// The default router can't be created, deleted or fetched, but its routes +/// work normally. +#[tokio::test] +#[ignore] +async fn test_default_router_is_not_a_router() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let cidr: Ipv4Net = "10.206.0.0/24".parse().unwrap(); + let target = ipv4_route(switch, 10, "10.10.10.1"); + + client.route_ipv4_add(&route_update(cidr, &target)).await?; + + let err = client.router_get(&DEFAULT_ROUTER).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::NOT_FOUND)); + let err = client + .router_create(&DEFAULT_ROUTER, &ipv6("fd00:206::1")) + .await + .unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::BAD_REQUEST)); + let err = client.router_delete(&DEFAULT_ROUTER).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::BAD_REQUEST)); + assert!(client.router_list().await?.is_empty()); + assert_eq!(client.route_ipv4_get(&cidr).await?.len(), 1); + + client.route_ipv4_delete(&cidr).await?; + Ok(()) +} + +/// v13 and v14 clients reach the same table through `/route/*`. +#[tokio::test] +#[ignore] +async fn test_legacy_route_uses_default_router() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let cidr: Ipv4Net = "10.205.0.0/24".parse().unwrap(); + let target = ipv4_route(switch, 10, "10.10.10.1"); + + let resp = reqwest::Client::new() + .post(format!("{}/route/ipv4", client.baseurl())) + .header("api-version", "13.0.0") + .json(&route_update(cidr, &target)) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let routes = client.route_ipv4_list(None, None).await?; + let found = routes.items.iter().find(|r| r.cidr == cidr).unwrap(); + assert_eq!(found.targets, vec![types::Route::V4(target)]); + + client.route_ipv4_delete(&cidr).await?; + Ok(()) +} + +/// An IPv6 loopback added by an API v13 client lands in the loopback list. +#[tokio::test] +#[ignore] +async fn test_legacy_loopback() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let lo = types::Ipv6Entry { + tag: "testing".into(), + addr: "fd00:207::1".parse().unwrap(), + }; + + let resp = reqwest::Client::new() + .post(format!("{}/loopback/ipv6", client.baseurl())) + .header("api-version", "13.0.0") + .json(&lo) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let loopbacks = client.loopback_ipv6_list().await?; + assert!(loopbacks.iter().any(|entry| entry.addr == lo.addr)); + assert!(client.router_list().await?.is_empty()); + + client.loopback_ipv6_delete(&lo.addr).await?; + Ok(()) +} + +/// A switch port's link address can't be any router's endpoint. +#[tokio::test] +#[ignore] +async fn test_link_address_is_not_endpoint() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + add_link_addr(switch).await?; + + let link_addr = ipv6(LINK_ADDR); + let err = + client.router_create(&Uuid::new_v4(), &link_addr).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::CONFLICT)); + assert!(client.router_list().await?.is_empty()); + Ok(()) +} + +/// Resetting the switch deletes the created routers, and their endpoints can +/// be claimed again. +#[tokio::test] +#[ignore] +async fn test_reset_removes_routers() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let router = Uuid::new_v4(); + let endpoint = ipv6("fd00:208::1"); + + client.router_create(&router, &endpoint).await?; + + client.reset_all().await?; + + assert!(client.router_list().await?.is_empty()); + let again = Uuid::new_v4(); + client.router_create(&again, &endpoint).await?; + + client.router_delete(&again).await?; + Ok(()) +} + +/// A router created after another was deleted reuses its routing table, but +/// none of its routes. +#[tokio::test] +#[ignore] +async fn test_recreated_router_has_no_routes() -> TestResult { + let switch = &*get_switch().await; + let client = &switch.client; + let cidr: Ipv4Net = "10.209.0.0/24".parse().unwrap(); + let target = ipv4_route(switch, 10, "10.10.10.1"); + let endpoint = ipv6("fd00:209::1"); + + // Each test starts from a reset switch, so both routers get the first + // table. + let first = Uuid::new_v4(); + client.router_create(&first, &endpoint).await?; + client.router_route_ipv4_add(&first, &route_update(cidr, &target)).await?; + client.router_delete(&first).await?; + + let second = Uuid::new_v4(); + client.router_create(&second, &endpoint).await?; + assert!( + client + .router_route_ipv4_list(&second, None, None) + .await? + .items + .is_empty() + ); + let err = client.router_route_ipv4_get(&second, &cidr).await.unwrap_err(); + assert_eq!(err.status(), Some(StatusCode::NOT_FOUND)); + + client.router_delete(&second).await?; + Ok(()) +} + +// Packet tests. A sled on `SLED_PORT` sends a Geneve packet to an address on +// the switch. The switch decapsulates it and routes the inner packet with +// the table that address selects: table 0 for the port's link address, or +// the table of the router whose endpoint it is. This is the +// same path as `nat::test_nat_egress`. + +const SLED_PORT: PhysPort = PhysPort(10); +const SLED_MAC: &str = "11:22:33:44:55:66"; +const SLED_IP: &str = "fd00:1122:7788:101::4"; +/// The link address of `SLED_PORT`. +const LINK_ADDR: &str = "fd00:1122:3344:101::5"; +const A_ENDPOINT: &str = "fd00:1122:3344:101::a"; +const B_ENDPOINT: &str = "fd00:1122:3344:101::b"; +const GW_MAC: &str = "02:aa:bb:cc:dd:ee"; + +async fn add_link_addr(switch: &Switch) -> TestResult { + let (port_id, link_id) = switch.link_id(SLED_PORT).unwrap(); + let entry = types::Ipv6Entry { + addr: LINK_ADDR.parse()?, + tag: switch.client.inner().tag.clone(), + }; + switch.client.link_ipv6_create(&port_id, &link_id, &entry).await?; + Ok(()) +} + +/// Route `cidr` out of `port` in `router`'s table. The port is made an +/// uplink, as decapsulated packets may only leave through uplinks. +async fn add_route( + switch: &Switch, + router: Option, + cidr: &str, + port: u16, +) -> TestResult { + let gw = format!("10.10.{port}.1"); + let target = ipv4_route(switch, port, &gw); + let update = route_update(cidr.parse()?, &target); + match router { + None => switch.client.route_ipv4_add(&update).await?, + Some(router) => { + switch.client.router_route_ipv4_add(&router, &update).await? + } + }; + common::add_arp_ipv4(switch, &gw, GW_MAC.parse()?).await?; + switch.set_uplink(PhysPort(port), true).await; + Ok(()) +} + +/// A UDP packet from the sled to `dst`. It carries the outer packet's +/// Ethernet header, since that's what decapsulation leaves. +fn inner_packet(switch: &Switch, dst: &str) -> Packet { + let switch_mac = switch.get_port_mac(SLED_PORT).unwrap().to_string(); + common::gen_udp_packet( + Endpoint::parse(SLED_MAC, "172.16.10.33", 3333).unwrap(), + Endpoint::parse(&switch_mac, dst, 4444).unwrap(), + ) +} + +/// `inner` sent by the sled in a Geneve packet to `outer_dst`. +fn geneve_to(switch: &Switch, outer_dst: &str, inner: &Packet) -> TestPacket { + let switch_mac = switch.get_port_mac(SLED_PORT).unwrap().to_string(); + let payload = inner.deparse().unwrap().to_vec(); + let packet = common::gen_geneve_packet( + Endpoint::parse(SLED_MAC, SLED_IP, 3333).unwrap(), + Endpoint::parse(&switch_mac, outer_dst, geneve::GENEVE_UDP_PORT) + .unwrap(), + eth::ETHER_IPV4, + 1, + &[], + &payload[14..], + ); + TestPacket { packet: Arc::new(packet), port: SLED_PORT } +} + +/// `inner` as it leaves `port` after being routed. +fn forwarded_packet(switch: &Switch, inner: &Packet, port: u16) -> TestPacket { + let mut packet = common::gen_packet_routed(switch, PhysPort(port), inner); + eth::EthHdr::rewrite_dmac(&mut packet, GW_MAC.parse().unwrap()); + TestPacket { packet: Arc::new(packet), port: PhysPort(port) } +} + +/// Create routers A and B with their endpoints. +async fn create_a_and_b( + switch: &Switch, +) -> Result<(Uuid, Uuid), anyhow::Error> { + let (a, b) = (Uuid::new_v4(), Uuid::new_v4()); + switch.client.router_create(&a, &ipv6(A_ENDPOINT)).await?; + switch.client.router_create(&b, &ipv6(B_ENDPOINT)).await?; + Ok((a, b)) +} + +/// One inner prefix, routed out of a different port by each router. +#[tokio::test] +#[ignore] +async fn test_endpoint_selects_router() -> TestResult { + let switch = &*get_switch().await; + add_link_addr(switch).await?; + let (a, b) = create_a_and_b(switch).await?; + let cidr = "10.10.10.0/24"; + add_route(switch, None, cidr, 14).await?; + add_route(switch, Some(a), cidr, 15).await?; + add_route(switch, Some(b), cidr, 16).await?; + + let inner = inner_packet(switch, "10.10.10.32"); + for (outer_dst, port) in + [(LINK_ADDR, 14), (A_ENDPOINT, 15), (B_ENDPOINT, 16)] + { + switch.packet_test( + vec![geneve_to(switch, outer_dst, &inner)], + vec![forwarded_packet(switch, &inner, port)], + )?; + } + Ok(()) +} + +/// A router doesn't fall back to the default router's routes. +#[tokio::test] +#[ignore] +async fn test_no_fallback_to_default_router() -> TestResult { + let switch = &*get_switch().await; + add_link_addr(switch).await?; + create_a_and_b(switch).await?; + add_route(switch, None, "10.10.10.0/24", 14).await?; + + let inner = inner_packet(switch, "10.10.10.32"); + let mut unreachable = inner.clone(); + common::set_icmp_unreachable(switch, &mut unreachable, SLED_PORT); + switch.packet_test( + vec![geneve_to(switch, A_ENDPOINT, &inner)], + vec![TestPacket { packet: Arc::new(unreachable), port: SERVICE_PORT }], + ) +} + +/// The longest matching prefix is chosen within one router's table only: a +/// longer prefix in another router's table doesn't win. +#[tokio::test] +#[ignore] +async fn test_longest_prefix_within_router() -> TestResult { + let switch = &*get_switch().await; + add_link_addr(switch).await?; + let (a, _) = create_a_and_b(switch).await?; + add_route(switch, None, "10.0.0.0/8", 14).await?; + add_route(switch, Some(a), "10.1.0.0/16", 15).await?; + + let inner = inner_packet(switch, "10.1.1.1"); + switch.packet_test( + vec![geneve_to(switch, LINK_ADDR, &inner)], + vec![forwarded_packet(switch, &inner, 14)], + ) +} diff --git a/dpd-client/tests/integration_tests/table_tests.rs b/dpd-client/tests/integration_tests/table_tests.rs index f0d4c4b8..61b074b9 100644 --- a/dpd-client/tests/integration_tests/table_tests.rs +++ b/dpd-client/tests/integration_tests/table_tests.rs @@ -46,9 +46,9 @@ use crate::integration_tests::common::prelude::*; // Investigate moving MulticastRouter4/6 into the egress pipeline to reclaim // capacity. #[cfg(feature = "multicast")] -const IPV4_LPM_SIZE: usize = 8145; // ipv4 forwarding table +const IPV4_LPM_SIZE: usize = 8153; // ipv4 forwarding table #[cfg(not(feature = "multicast"))] -const IPV4_LPM_SIZE: usize = 8191; // ipv4 forwarding table +const IPV4_LPM_SIZE: usize = 8190; // ipv4 forwarding table #[cfg(feature = "multicast")] const IPV6_LPM_SIZE: usize = 1023; // ipv6 forwarding table diff --git a/dpd-types/versions/src/latest.rs b/dpd-types/versions/src/latest.rs index 3b56eb39..eb145887 100644 --- a/dpd-types/versions/src/latest.rs +++ b/dpd-types/versions/src/latest.rs @@ -171,11 +171,8 @@ pub mod route { pub use crate::v1::route::Ipv6RouteToken; pub use crate::v1::route::Ipv6RouteUpdate; pub use crate::v1::route::Ipv6Routes; - pub use crate::v1::route::RoutePathV4; - pub use crate::v1::route::RoutePathV6; pub use crate::v1::route::RouteSettingsV4; pub use crate::v1::route::RouteSettingsV6; - pub use crate::v1::route::RouteTargetIpv6Path; pub use crate::v3::route::AttachedSubnetToken; pub use crate::v3::route::SubnetPath; @@ -185,7 +182,13 @@ pub mod route { pub use crate::v4::route::RouteTarget; pub use crate::v6::route::Ipv4RouteUpdate; - pub use crate::v6::route::RouteTargetIpv4Path; + + pub use crate::v14::route::DEFAULT_ROUTER; + pub use crate::v14::route::RoutePathV4; + pub use crate::v14::route::RoutePathV6; + pub use crate::v14::route::RouteTargetIpv4Path; + pub use crate::v14::route::RouteTargetIpv6Path; + pub use crate::v14::route::RouterPath; } pub mod serdes { diff --git a/dpd-types/versions/src/lib.rs b/dpd-types/versions/src/lib.rs index 617a58c7..d7a88d66 100644 --- a/dpd-types/versions/src/lib.rs +++ b/dpd-types/versions/src/lib.rs @@ -43,6 +43,8 @@ pub mod v11; pub mod v12; #[path = "allow_ddm_traffic/mod.rs"] pub mod v13; +#[path = "modular_router/mod.rs"] +pub mod v14; #[path = "attached_subnets/mod.rs"] pub mod v3; #[path = "v4_over_v6_routes/mod.rs"] diff --git a/dpd-types/versions/src/modular_router/mod.rs b/dpd-types/versions/src/modular_router/mod.rs new file mode 100644 index 00000000..fdf701d6 --- /dev/null +++ b/dpd-types/versions/src/modular_router/mod.rs @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +//! Version `MODULAR_ROUTER` of the DPD API. +//! +//! Adds routers, each with its own routes and endpoint, under +//! `/router/{router_id}/...`. The `/route/...` endpoints act on the default +//! routing table. + +pub mod route; diff --git a/dpd-types/versions/src/modular_router/route.rs b/dpd-types/versions/src/modular_router/route.rs new file mode 100644 index 00000000..4631f4a6 --- /dev/null +++ b/dpd-types/versions/src/modular_router/route.rs @@ -0,0 +1,113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use std::net::{IpAddr, Ipv6Addr}; + +use crate::v1; +use crate::v1::port::PortId; +use crate::v6; +use oxnet::{Ipv4Net, Ipv6Net}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::v1::link::LinkId; + +/// The logical router used for backwards compatibility, sharing routing table +/// 0 with the underlay routes. This is to be removed in a subsequent version. +pub const DEFAULT_ROUTER: Uuid = Uuid::nil(); + +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct RouterPath { + /// The router being addressed. + pub router_id: Uuid, +} + +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct RoutePathV4 { + /// The router being addressed. + pub router_id: Uuid, + /// The IPv4 subnet in CIDR notation whose route entry is returned. + pub cidr: Ipv4Net, +} + +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct RoutePathV6 { + /// The router being addressed. + pub router_id: Uuid, + /// The IPv6 subnet in CIDR notation whose route entry is returned. + pub cidr: Ipv6Net, +} + +/// Represents a single subnet->target route entry with an IPv4 or IPv6 +/// next hop. +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct RouteTargetIpv4Path { + /// The router being addressed. + pub router_id: Uuid, + /// The subnet being routed + pub cidr: Ipv4Net, + /// The switch port to which packets should be sent + pub port_id: PortId, + /// The link to which packets should be sent + pub link_id: LinkId, + /// The next hop in the route (IPv4 or IPv6) + pub tgt_ip: IpAddr, +} + +/// Represents a single subnet->target route entry +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct RouteTargetIpv6Path { + /// The router being addressed. + pub router_id: Uuid, + /// The subnet being routed + pub cidr: Ipv6Net, + /// The switch port to which packets should be sent + pub port_id: PortId, + /// The link to which packets should be sent + pub link_id: LinkId, + /// The next hop in the IPv6 route + pub tgt_ip: Ipv6Addr, +} + +// Paths from earlier versions have no router, and address the default routing +// table. + +impl From for RoutePathV4 { + fn from(old: v1::route::RoutePathV4) -> Self { + Self { router_id: DEFAULT_ROUTER, cidr: old.cidr } + } +} + +impl From for RoutePathV6 { + fn from(old: v1::route::RoutePathV6) -> Self { + Self { router_id: DEFAULT_ROUTER, cidr: old.cidr } + } +} + +impl From for RouteTargetIpv4Path { + fn from(old: v6::route::RouteTargetIpv4Path) -> Self { + Self { + router_id: DEFAULT_ROUTER, + cidr: old.cidr, + port_id: old.port_id, + link_id: old.link_id, + tgt_ip: old.tgt_ip, + } + } +} + +impl From for RouteTargetIpv6Path { + fn from(old: v1::route::RouteTargetIpv6Path) -> Self { + Self { + router_id: DEFAULT_ROUTER, + cidr: old.cidr, + port_id: old.port_id, + link_id: old.link_id, + tgt_ip: old.tgt_ip, + } + } +} diff --git a/dpd/p4/metadata.p4 b/dpd/p4/metadata.p4 index 8fcbda5f..5981906e 100644 --- a/dpd/p4/metadata.p4 +++ b/dpd/p4/metadata.p4 @@ -29,6 +29,7 @@ struct sidecar_ingress_meta_t { ipv6_addr_t nexthop_ipv6; // ip address of next router bit<10> pkt_type; bit<8> drop_reason; // reason a packet was dropped + bit<8> routing_table_id; // routing table to look up routes in bit<16> l4_src_port; // tcp or udp destination port bit<16> l4_dst_port; // tcp or udp destination port bit<16> flow_hash; // hash of a packet 5-tuple for ECMP/encap diff --git a/dpd/p4/parser.p4 b/dpd/p4/parser.p4 index 07af7074..0f83248a 100644 --- a/dpd/p4/parser.p4 +++ b/dpd/p4/parser.p4 @@ -51,6 +51,7 @@ parser IngressParser( meta.orig_dst_ipv4 = 0; meta.pkt_type = 0; meta.drop_reason = 0; + meta.routing_table_id = 0; meta.nat_ingress_csum = 0; meta.resolve_nexthop = false; diff --git a/dpd/p4/sidecar.p4 b/dpd/p4/sidecar.p4 index 108dba15..29ed4b5e 100644 --- a/dpd/p4/sidecar.p4 +++ b/dpd/p4/sidecar.p4 @@ -132,12 +132,12 @@ control Filter( ipv4_ctr.count(); } - action claimv6() { + action claimv6(bit<8> routing_table_id) { meta.is_switch_address = true; + meta.routing_table_id = routing_table_id; ipv6_ctr.count(); } - // Table of the IPv4 addresses assigned to ports on the switch. table switch_ipv4_addr { key = { @@ -894,6 +894,7 @@ control NatEgress ( control RouterLookupIndex6( inout sidecar_headers_t hdr, + in sidecar_ingress_meta_t meta, inout route6_result_t res ) { DirectCounter>(CounterType_t.PACKETS_AND_BYTES) index_ctr; @@ -962,7 +963,10 @@ control RouterLookupIndex6( } table lookup { - key = { hdr.ipv6.dst_addr: lpm; } + key = { + meta.routing_table_id: exact; + hdr.ipv6.dst_addr: lpm; + } actions = { index; unreachable; } default_action = unreachable; // The table size is incremented by one here just to allow the @@ -992,6 +996,7 @@ control RouterLookupIndex6( control RouterLookupIndex4( inout sidecar_headers_t hdr, + in sidecar_ingress_meta_t meta, inout route4_result_t res ) { DirectCounter>(CounterType_t.PACKETS_AND_BYTES) index_ctr; @@ -1083,7 +1088,10 @@ control RouterLookupIndex4( } table lookup { - key = { hdr.ipv4.dst_addr: lpm; } + key = { + meta.routing_table_id: exact; + hdr.ipv4.dst_addr: lpm; + } actions = { index; unreachable; } default_action = unreachable; const size = IPV4_LPM_SIZE; @@ -1240,7 +1248,7 @@ control Router4 ( // 11-bit index. fwd.ecmp_hash = 2w0 ++ meta.flow_hash[5:0]; - lookup_idx.apply(hdr, fwd); + lookup_idx.apply(hdr, meta, fwd); if (!fwd.is_hit) { icmp_error(ICMP_DEST_UNREACH, ICMP_DST_UNREACH_NET); @@ -1379,7 +1387,7 @@ control Router6 ( // 11-bit index. fwd.ecmp_hash = 2w0 ++ meta.flow_hash[5:0]; - lookup_idx.apply(hdr, fwd); + lookup_idx.apply(hdr, meta, fwd); if (!fwd.is_hit) { icmp_error(ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_NOROUTE); diff --git a/dpd/src/api_server.rs b/dpd/src/api_server.rs index 5f595e6c..b4af6a11 100644 --- a/dpd/src/api_server.rs +++ b/dpd/src/api_server.rs @@ -51,10 +51,10 @@ use dpd_types::port::{ }; use dpd_types::port_map::BackplaneLink; use dpd_types::route::{ - AttachedSubnetToken, Ipv4RouteToken, Ipv4RouteUpdate, Ipv4Routes, - Ipv6Route, Ipv6RouteToken, Ipv6RouteUpdate, Ipv6Routes, Route, RoutePathV4, - RoutePathV6, RouteTarget, RouteTargetIpv4Path, RouteTargetIpv6Path, - SubnetPath, + AttachedSubnetToken, DEFAULT_ROUTER, Ipv4RouteToken, Ipv4RouteUpdate, + Ipv4Routes, Ipv6Route, Ipv6RouteToken, Ipv6RouteUpdate, Ipv6Routes, Route, + RoutePathV4, RoutePathV6, RouteTarget, RouteTargetIpv4Path, + RouteTargetIpv6Path, RouterPath, SubnetPath, }; use dpd_types::serdes::{ AnLtStatus, Ber, DfeAdaptationState, EncSpeed, LaneMap, RxSigInfo, @@ -69,7 +69,7 @@ use dpd_types::switch_port::{Led, LedState, ManagementMode, SwitchPortView}; use dpd_types::table; use dpd_types::table::TableParam; use dpd_types::transceivers::{Datapath, Monitors, PowerState, Transceiver}; -use dpd_types_versions::{v1, v7}; +use dpd_types_versions::{v1, v6, v7}; use dropshot::BuildError; use dropshot::ClientErrorStatusCode; use dropshot::ClientSpecifiesVersionInHeader; @@ -104,10 +104,12 @@ use crate::counters; use crate::mcast; use crate::nat; use crate::oxstats; +use crate::router::RouterUuid; +use crate::router::RoutingTableId; use crate::rpw::Task; use crate::switch_port::FixedSideDevice; use crate::types::DpdError; -use crate::{Switch, arp, loopback, ports, route}; +use crate::{Switch, arp, loopback, ports, route, router}; use common::attached_subnet::AttachedSubnetEntry; use common::nat::{Ipv4Nat, Ipv6Nat}; use common::network::{InstanceTarget, MacAddr, NatTarget}; @@ -115,6 +117,7 @@ use common::ports::PortId; use common::ports::QsfpPort; use common::ports::{Ipv4Entry, Ipv6Entry, PortPrbsMode}; use dpd_api::*; +use uuid::Uuid; type ApiServer = dropshot::HttpServer>; @@ -333,204 +336,90 @@ impl DpdApi for DpdApiImpl { rqctx: RequestContext>, query: Query>, ) -> Result>, HttpError> { - let switch: &Switch = rqctx.context(); - let pag_params = query.into_inner(); - let max = rqctx.page_limit(&pag_params)?.get(); - - let previous = match &pag_params.page { - WhichPage::First(..) => None, - WhichPage::Next(Ipv6RouteToken { cidr }) => Some(*cidr), - }; - - route::get_range_ipv6(switch, previous, max) - .await - .map_err(HttpError::from) - .and_then(|entries| { - ResultsPage::new( - entries, - &EmptyScanParams {}, - |e: &Ipv6Routes, _| Ipv6RouteToken { cidr: e.cidr }, - ) - }) - .map(HttpResponseOk) + let path = RouterPath { router_id: DEFAULT_ROUTER }; + Self::router_route_ipv6_list(rqctx, path.into(), query).await } async fn route_ipv6_get( rqctx: RequestContext>, - path: Path, + path: Path, ) -> Result>, HttpError> { - let switch: &Switch = rqctx.context(); - let cidr = path.into_inner().cidr; - route::get_route_ipv6(switch, cidr) - .await - .map(HttpResponseOk) - .map_err(HttpError::from) + Self::router_route_ipv6_get(rqctx, path.map(Into::into)).await } async fn route_ipv6_add( rqctx: RequestContext>, update: TypedBody, ) -> Result { - let switch: &Switch = rqctx.context(); - let route = update.into_inner(); - route::add_route_ipv6(switch, route.cidr, route.target) - .await - .map(|_| HttpResponseUpdatedNoContent()) - .map_err(HttpError::from) + let path = RouterPath { router_id: DEFAULT_ROUTER }; + Self::router_route_ipv6_add(rqctx, path.into(), update).await } async fn route_ipv6_set( rqctx: RequestContext>, update: TypedBody, ) -> Result { - let switch: &Switch = rqctx.context(); - let route = update.into_inner(); - route::set_route_ipv6(switch, route.cidr, route.target, route.replace) - .await - .map(|_| HttpResponseUpdatedNoContent()) - .map_err(HttpError::from) + let path = RouterPath { router_id: DEFAULT_ROUTER }; + Self::router_route_ipv6_set(rqctx, path.into(), update).await } async fn route_ipv6_delete( rqctx: RequestContext>, - path: Path, + path: Path, ) -> Result { - let switch: &Switch = rqctx.context(); - let cidr = path.into_inner().cidr; - route::delete_route_ipv6(switch, cidr) - .await - .map(|_| HttpResponseDeleted()) - .map_err(HttpError::from) + Self::router_route_ipv6_delete(rqctx, path.map(Into::into)).await } async fn route_ipv6_delete_target( rqctx: RequestContext>, - path: Path, + path: Path, ) -> Result { - let switch: &Switch = rqctx.context(); - let path = path.into_inner(); - let subnet = path.cidr; - let port_id = path.port_id; - let link_id = path.link_id; - let tgt_ip = path.tgt_ip; - route::delete_route_target_ipv6( - switch, subnet, port_id, link_id, tgt_ip, - ) - .await - .map(|_| HttpResponseDeleted()) - .map_err(HttpError::from) + Self::router_route_ipv6_delete_target(rqctx, path.map(Into::into)).await } async fn route_ipv4_list( rqctx: RequestContext>, query: Query>, ) -> Result>, HttpError> { - let switch: &Switch = rqctx.context(); - let pag_params = query.into_inner(); - let max = rqctx.page_limit(&pag_params)?.get(); - - let previous = match &pag_params.page { - WhichPage::First(..) => None, - WhichPage::Next(Ipv4RouteToken { cidr }) => Some(*cidr), - }; - - route::get_range_ipv4(switch, previous, max) - .await - .map_err(HttpError::from) - .and_then(|entries| { - ResultsPage::new( - entries, - &EmptyScanParams {}, - |e: &Ipv4Routes, _| Ipv4RouteToken { cidr: e.cidr }, - ) - }) - .map(HttpResponseOk) + let path = RouterPath { router_id: DEFAULT_ROUTER }; + Self::router_route_ipv4_list(rqctx, path.into(), query).await } async fn route_ipv4_get( rqctx: RequestContext>, - path: Path, + path: Path, ) -> Result>, HttpError> { - let switch: &Switch = rqctx.context(); - let cidr = path.into_inner().cidr; - route::get_route_ipv4(switch, cidr) - .await - .map(HttpResponseOk) - .map_err(HttpError::from) + Self::router_route_ipv4_get(rqctx, path.map(Into::into)).await } async fn route_ipv4_add( rqctx: RequestContext>, update: TypedBody, ) -> Result { - let switch: &Switch = rqctx.context(); - let route = update.into_inner(); - match route.target { - RouteTarget::V4(target) => { - route::add_route_ipv4(switch, route.cidr, target).await - } - RouteTarget::V6(target) => { - route::add_route_ipv4_over_ipv6(switch, route.cidr, target) - .await - } - } - .map(|_| HttpResponseUpdatedNoContent()) - .map_err(HttpError::from) + let path = RouterPath { router_id: DEFAULT_ROUTER }; + Self::router_route_ipv4_add(rqctx, path.into(), update).await } async fn route_ipv4_set( rqctx: RequestContext>, update: TypedBody, ) -> Result { - let switch: &Switch = rqctx.context(); - let route = update.into_inner(); - match route.target { - RouteTarget::V4(target) => { - route::set_route_ipv4(switch, route.cidr, target, route.replace) - .await - } - RouteTarget::V6(target) => { - route::set_route_ipv4_over_ipv6( - switch, - route.cidr, - target, - route.replace, - ) - .await - } - } - .map(|_| HttpResponseUpdatedNoContent()) - .map_err(HttpError::from) + let path = RouterPath { router_id: DEFAULT_ROUTER }; + Self::router_route_ipv4_set(rqctx, path.into(), update).await } async fn route_ipv4_delete( rqctx: RequestContext>, - path: Path, + path: Path, ) -> Result { - let switch: &Switch = rqctx.context(); - let cidr = path.into_inner().cidr; - route::delete_route_ipv4(switch, cidr) - .await - .map(|_| HttpResponseDeleted()) - .map_err(HttpError::from) + Self::router_route_ipv4_delete(rqctx, path.map(Into::into)).await } async fn route_ipv4_delete_target( rqctx: RequestContext>, - path: Path, + path: Path, ) -> Result { - let switch: &Switch = rqctx.context(); - let path = path.into_inner(); - route::delete_route_target_ipv4( - switch, - path.cidr, - path.port_id, - path.link_id, - path.tgt_ip, - ) - .await - .map(|_| HttpResponseDeleted()) - .map_err(HttpError::from) + Self::router_route_ipv4_delete_target(rqctx, path.map(Into::into)).await } async fn port_list( @@ -1388,8 +1277,10 @@ impl DpdApi for DpdApiImpl { ) -> Result { let switch: &Switch = rqctx.context(); let addr = val.into_inner(); - - loopback::add_loopback_ipv6(switch, &addr)?; + // We refuse loopbacks that are already taken by routers. + let route_data = switch.routes.lock().await; + route_data.routers.check_not_endpoint(addr.addr)?; + loopback::add_loopback_ipv6(switch, &addr, RoutingTableId::default())?; Ok(HttpResponseUpdatedNoContent {}) } @@ -1400,11 +1291,301 @@ impl DpdApi for DpdApiImpl { ) -> Result { let switch: &Switch = rqctx.context(); let addr = path.into_inner(); + // A router's endpoint is only removed when deleting the router. + let route_data = switch.routes.lock().await; + route_data.routers.check_not_endpoint(addr.ipv6)?; loopback::delete_loopback_ipv6(switch, &addr.ipv6) .map(|_| HttpResponseDeleted()) .map_err(HttpError::from) } + async fn router_list( + rqctx: RequestContext>, + ) -> Result>, HttpError> { + let switch: &Switch = rqctx.context(); + Ok(HttpResponseOk(switch.routes.lock().await.routers.list())) + } + + async fn router_get( + rqctx: RequestContext>, + path: Path, + ) -> Result, HttpError> { + let switch: &Switch = rqctx.context(); + let route_data = switch.routes.lock().await; + route_data + .routers + .endpoint(path.into_inner().router_id.into()) + .map(HttpResponseOk) + .map_err(HttpError::from) + } + + async fn router_create( + rqctx: RequestContext>, + path: Path, + endpoint: TypedBody, + ) -> Result { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let mut route_data = switch.routes.lock().await; + route_data + .routers + .create(switch, router, endpoint.into_inner()) + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn router_delete( + rqctx: RequestContext>, + path: Path, + ) -> Result { + let switch: &Switch = rqctx.context(); + let mut route_data = switch.routes.lock().await; + router::delete( + switch, + &mut route_data, + path.into_inner().router_id.into(), + ) + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + + async fn router_route_ipv6_list( + rqctx: RequestContext>, + path: Path, + query: Query>, + ) -> Result>, HttpError> { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let pag_params = query.into_inner(); + let max = rqctx.page_limit(&pag_params)?.get(); + + let previous = match &pag_params.page { + WhichPage::First(..) => None, + WhichPage::Next(Ipv6RouteToken { cidr }) => Some(*cidr), + }; + + route::get_range_ipv6(switch, router, previous, max) + .await + .map_err(HttpError::from) + .and_then(|entries| { + ResultsPage::new( + entries, + &EmptyScanParams {}, + |e: &Ipv6Routes, _| Ipv6RouteToken { cidr: e.cidr }, + ) + }) + .map(HttpResponseOk) + } + + async fn router_route_ipv6_get( + rqctx: RequestContext>, + path: Path, + ) -> Result>, HttpError> { + let switch: &Switch = rqctx.context(); + let path = path.into_inner(); + route::get_route_ipv6(switch, path.router_id.into(), path.cidr) + .await + .map(HttpResponseOk) + .map_err(HttpError::from) + } + + async fn router_route_ipv6_add( + rqctx: RequestContext>, + path: Path, + update: TypedBody, + ) -> Result { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let route = update.into_inner(); + route::add_route_ipv6(switch, router, route.cidr, route.target) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn router_route_ipv6_set( + rqctx: RequestContext>, + path: Path, + update: TypedBody, + ) -> Result { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let route = update.into_inner(); + route::set_route_ipv6( + switch, + router, + route.cidr, + route.target, + route.replace, + ) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn router_route_ipv6_delete( + rqctx: RequestContext>, + path: Path, + ) -> Result { + let switch: &Switch = rqctx.context(); + let path = path.into_inner(); + route::delete_route_ipv6(switch, path.router_id.into(), path.cidr) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + + async fn router_route_ipv6_delete_target( + rqctx: RequestContext>, + path: Path, + ) -> Result { + let switch: &Switch = rqctx.context(); + let path = path.into_inner(); + let subnet = path.cidr; + let port_id = path.port_id; + let link_id = path.link_id; + let tgt_ip = path.tgt_ip; + route::delete_route_target_ipv6( + switch, + path.router_id.into(), + subnet, + port_id, + link_id, + tgt_ip, + ) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + + async fn router_route_ipv4_list( + rqctx: RequestContext>, + path: Path, + query: Query>, + ) -> Result>, HttpError> { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let pag_params = query.into_inner(); + let max = rqctx.page_limit(&pag_params)?.get(); + + let previous = match &pag_params.page { + WhichPage::First(..) => None, + WhichPage::Next(Ipv4RouteToken { cidr }) => Some(*cidr), + }; + + route::get_range_ipv4(switch, router, previous, max) + .await + .map_err(HttpError::from) + .and_then(|entries| { + ResultsPage::new( + entries, + &EmptyScanParams {}, + |e: &Ipv4Routes, _| Ipv4RouteToken { cidr: e.cidr }, + ) + }) + .map(HttpResponseOk) + } + + async fn router_route_ipv4_get( + rqctx: RequestContext>, + path: Path, + ) -> Result>, HttpError> { + let switch: &Switch = rqctx.context(); + let path = path.into_inner(); + route::get_route_ipv4(switch, path.router_id.into(), path.cidr) + .await + .map(HttpResponseOk) + .map_err(HttpError::from) + } + + async fn router_route_ipv4_add( + rqctx: RequestContext>, + path: Path, + update: TypedBody, + ) -> Result { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let route = update.into_inner(); + match route.target { + RouteTarget::V4(target) => { + route::add_route_ipv4(switch, router, route.cidr, target).await + } + RouteTarget::V6(target) => { + route::add_route_ipv4_over_ipv6( + switch, router, route.cidr, target, + ) + .await + } + } + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn router_route_ipv4_set( + rqctx: RequestContext>, + path: Path, + update: TypedBody, + ) -> Result { + let switch: &Switch = rqctx.context(); + let router = RouterUuid::from(path.into_inner().router_id); + let route = update.into_inner(); + match route.target { + RouteTarget::V4(target) => { + route::set_route_ipv4( + switch, + router, + route.cidr, + target, + route.replace, + ) + .await + } + RouteTarget::V6(target) => { + route::set_route_ipv4_over_ipv6( + switch, + router, + route.cidr, + target, + route.replace, + ) + .await + } + } + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + } + + async fn router_route_ipv4_delete( + rqctx: RequestContext>, + path: Path, + ) -> Result { + let switch: &Switch = rqctx.context(); + let path = path.into_inner(); + route::delete_route_ipv4(switch, path.router_id.into(), path.cidr) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + + async fn router_route_ipv4_delete_target( + rqctx: RequestContext>, + path: Path, + ) -> Result { + let switch: &Switch = rqctx.context(); + let path = path.into_inner(); + route::delete_route_target_ipv4( + switch, + path.router_id.into(), + path.cidr, + path.port_id, + path.link_id, + path.tgt_ip, + ) + .await + .map(|_| HttpResponseDeleted()) + .map_err(HttpError::from) + } + async fn nat_ipv6_addresses_list( rqctx: RequestContext>, query: Query>, @@ -1730,6 +1911,11 @@ impl DpdApi for DpdApiImpl { error!(switch.log, "failed to reset ipv6 arp table: {:?}", e); err = Some(e); } + if let Err(e) = router::reset(switch, &mut *switch.routes.lock().await) + { + error!(switch.log, "failed to reset routers: {:?}", e); + err = Some(e); + } if let Err(e) = route::reset(switch).await { error!(switch.log, "failed to reset route data: {:?}", e); err = Some(e); diff --git a/dpd/src/loopback.rs b/dpd/src/loopback.rs index f0a4665c..c8aa1b98 100644 --- a/dpd/src/loopback.rs +++ b/dpd/src/loopback.rs @@ -4,6 +4,7 @@ // // Copyright 2026 Oxide Computer Company +use crate::router::RoutingTableId; use crate::{DpdError, DpdResult, Switch, table}; use aal::AsicError; use common::ports::{Ipv4Entry, Ipv6Entry}; @@ -73,16 +74,30 @@ pub fn delete_loopback_ipv4(switch: &Switch, addr: &Ipv4Addr) -> DpdResult<()> { } /// Add a loopback IPv6 address to the switch. -pub fn add_loopback_ipv6(switch: &Switch, addr: &Ipv6Entry) -> DpdResult<()> { +/// +/// We won't "inherit" a loopback if we're adding a new endpoint. +pub fn add_loopback_ipv6( + switch: &Switch, + addr: &Ipv6Entry, + table_id: RoutingTableId, +) -> DpdResult<()> { let mut loopback_data = switch.loopback.lock().unwrap(); if loopback_data.v6_addrs.contains(addr) { + if table_id != RoutingTableId::default() { + return Err(DpdError::Exists(format!( + "{} is already a loopback", + addr.addr + ))); + } debug!(switch.log, "loopback entry {} already set", addr.addr); return Ok(()); } - match table::port_ip::loopback_ipv6_add(switch, addr.addr) { + match table::port_ip::loopback_ipv6_add(switch, addr.addr, table_id) { Ok(()) => _ = loopback_data.v6_addrs.insert(addr.clone()), - Err(DpdError::Switch(AsicError::Exists)) => { + Err(DpdError::Switch(AsicError::Exists)) + if table_id == RoutingTableId::default() => + { if !loopback_data.v6_addrs.contains(addr) { warn!( switch.log, diff --git a/dpd/src/main.rs b/dpd/src/main.rs index 5fc08ec5..6990a934 100644 --- a/dpd/src/main.rs +++ b/dpd/src/main.rs @@ -73,6 +73,7 @@ mod port_map; mod port_settings; mod ports; mod route; +mod router; mod rpw; #[cfg(feature = "tofino_asic")] mod snapshot; @@ -191,6 +192,7 @@ pub struct Switch { pub tables: BTreeMap>, pub counters: BTreeMap>, pub links: Mutex, + // This should be locked before `loopback`. pub routes: TokioMutex, pub arp: Mutex, pub nat: nat::Nat, diff --git a/dpd/src/route.rs b/dpd/src/route.rs index fcbc3854..42305ee0 100644 --- a/dpd/src/route.rs +++ b/dpd/src/route.rs @@ -110,6 +110,7 @@ use std::convert::TryFrom; use std::net::{IpAddr, Ipv6Addr}; use std::ops::Bound; +use aal::AsicError; use dpd_types::link::LinkId; use dpd_types::route::Ipv4Route; use dpd_types::route::Ipv6Route; @@ -119,6 +120,7 @@ use slog::info; use slog::warn; use crate::freemap; +use crate::router::{RouterUuid, Routers, RoutingTableId}; use crate::types::{DpdError, DpdResult}; use crate::{Switch, table}; use common::ports::PortId; @@ -237,6 +239,25 @@ impl From for Route { } } +/// Identifies a route by its routing table and subnet. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct RouteKey { + table_id: RoutingTableId, + subnet: IpNet, +} + +impl RouteKey { + fn new(table_id: RoutingTableId, subnet: impl Into) -> Self { + RouteKey { table_id, subnet: subnet.into() } + } +} + +impl std::fmt::Display for RouteKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} (routing table {})", self.subnet, self.table_id) + } +} + #[derive(Debug, Eq, PartialEq, Clone)] struct NextHop { asic_port_id: u16, @@ -257,9 +278,18 @@ impl RouteEntry { } } -pub struct RouteData { +#[derive(Default)] +struct RoutingTable { v4: BTreeMap, v6: BTreeMap, +} + +/// The routers and routing tables. +pub struct RouteData { + /// The routers, kept here so that a router and its table change together. + pub routers: Routers, + /// The routing tables, keyed by `RoutingTableId`. + tables: BTreeMap, v4_freemap: freemap::FreeMap, v6_freemap: freemap::FreeMap, } @@ -267,35 +297,59 @@ pub struct RouteData { impl RouteData { pub fn insert( &mut self, - subnet: impl Into, + key: RouteKey, entry: RouteEntry, ) -> Option { - let subnet: IpNet = subnet.into(); - if subnet.is_ipv4() { - self.v4.insert(subnet, entry) + let table = self.tables.entry(key.table_id).or_default(); + if key.subnet.is_ipv4() { + table.v4.insert(key.subnet, entry) } else { - self.v6.insert(subnet, entry) + table.v6.insert(key.subnet, entry) } } - pub fn get(&self, subnet: impl Into) -> Option<&RouteEntry> { - let subnet: IpNet = subnet.into(); - if subnet.is_ipv4() { - self.v4.get(&subnet) + pub fn get(&self, key: RouteKey) -> Option<&RouteEntry> { + let table = self.tables.get(&key.table_id)?; + if key.subnet.is_ipv4() { + table.v4.get(&key.subnet) } else { - self.v6.get(&subnet) + table.v6.get(&key.subnet) } } - pub fn remove(&mut self, subnet: impl Into) -> Option { - let subnet: IpNet = subnet.into(); - if subnet.is_ipv4() { - self.v4.remove(&subnet) + pub fn remove(&mut self, key: RouteKey) -> Option { + let table = self.tables.get_mut(&key.table_id)?; + if key.subnet.is_ipv4() { + table.v4.remove(&key.subnet) } else { - self.v6.remove(&subnet) + table.v6.remove(&key.subnet) } } + /// Remove every route in routing table `table_id` from the switch, then + /// forget the table. If a removal fails, the remaining routes are kept so + /// the delete can be retried. + fn delete_table( + &mut self, + ops: &dyn RouteTableOps, + table_id: RoutingTableId, + ) -> DpdResult<()> { + let Some(table) = self.tables.get(&table_id) else { + return Ok(()); + }; + let keys: Vec = table + .v4 + .keys() + .chain(table.v6.keys()) + .map(|subnet| RouteKey::new(table_id, *subnet)) + .collect(); + for key in keys { + delete_route_locked(ops, self, key)?; + } + self.tables.remove(&table_id); + Ok(()) + } + fn freemap_mut(&mut self, is_ipv4: bool) -> &mut freemap::FreeMap { if is_ipv4 { &mut self.v4_freemap } else { &mut self.v6_freemap } } @@ -327,11 +381,11 @@ trait RouteTableOps { fn delete_target(&self, subnet: IpNet, idx: u16) -> DpdResult<()>; /// Install a `route_index` entry pointing at `[index, index + slots)` - /// of the family inferred from `subnet`. - fn add_index(&self, subnet: IpNet, index: u16, slots: u8) -> DpdResult<()>; + /// of the family inferred from `key`. + fn add_index(&self, key: RouteKey, index: u16, slots: u8) -> DpdResult<()>; - /// Delete the `route_index` entry for `subnet`. - fn delete_index(&self, subnet: IpNet) -> DpdResult<()>; + /// Delete the `route_index` entry for `key`. + fn delete_index(&self, key: RouteKey) -> DpdResult<()>; } impl RouteTableOps for Switch { @@ -383,27 +437,38 @@ impl RouteTableOps for Switch { } } - fn add_index(&self, subnet: IpNet, index: u16, slots: u8) -> DpdResult<()> { - match subnet { + fn add_index(&self, key: RouteKey, index: u16, slots: u8) -> DpdResult<()> { + match key.subnet { + IpNet::V4(v4) => table::route_ipv4::add_route_index( + self, + key.table_id, + &v4, + index, + slots, + ), + IpNet::V6(v6) => table::route_ipv6::add_route_index( + self, + key.table_id, + &v6, + index, + slots, + ), + } + } + + fn delete_index(&self, key: RouteKey) -> DpdResult<()> { + match key.subnet { IpNet::V4(v4) => { - table::route_ipv4::add_route_index(self, &v4, index, slots) + table::route_ipv4::delete_route_index(self, key.table_id, &v4) } IpNet::V6(v6) => { - table::route_ipv6::add_route_index(self, &v6, index, slots) + table::route_ipv6::delete_route_index(self, key.table_id, &v6) } } } - - fn delete_index(&self, subnet: IpNet) -> DpdResult<()> { - match subnet { - IpNet::V4(v4) => table::route_ipv4::delete_route_index(self, &v4), - IpNet::V6(v6) => table::route_ipv6::delete_route_index(self, &v6), - } - } } -// Remove all the data for a given route from both the route_data and -// route_index tables. +// Remove all the data for a given route from the route_data table. // // Because this may be called from the error-recovery path of a failed add-target // operation, not all of the target slots may yet be populated. Thus we require @@ -411,21 +476,16 @@ impl RouteTableOps for Switch { fn cleanup_route( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, - delete_index: bool, + key: RouteKey, entry: RouteEntry, ) -> DpdResult<()> { - // Remove the subnet -> index mapping first, so nobody can reach the - // entries we delete below. - if delete_index { - ops.delete_index(subnet)?; - } - let all_clear = entry .targets .iter() .enumerate() - .map(|(idx, _hop)| ops.delete_target(subnet, entry.index + idx as u16)) + .map(|(idx, _hop)| { + ops.delete_target(key.subnet, entry.index + idx as u16) + }) .all(|rval| rval.is_ok()); // If all of the entries were removed, we can release the table space back @@ -445,17 +505,17 @@ fn cleanup_route( fn finalize_route( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, entry: Option, ) -> DpdResult<()> { let Some(entry) = entry else { return Ok(()) }; - match ops.add_index(subnet, entry.index, entry.slots) { + match ops.add_index(key, entry.index, entry.slots) { Ok(_) => { - route_data.insert(subnet, entry); + route_data.insert(key, entry); Ok(()) } - Err(_) => cleanup_route(ops, route_data, subnet, false, entry), + Err(_) => cleanup_route(ops, route_data, key, entry), } } @@ -529,17 +589,17 @@ fn classify_update( fn unhook_route( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, ) -> DpdResult> { - let Some(old) = route_data.remove(subnet) else { + let Some(old) = route_data.remove(key) else { return Ok(None); }; - if let Err(e) = ops.delete_index(subnet) { + if let Err(e) = ops.delete_index(key) { debug!( ops.log(), - "unhook_route: route_index delete failed for {subnet}: {e:?}" + "unhook_route: route_index delete failed for {key}: {e:?}" ); - route_data.insert(subnet, old); + route_data.insert(key, old); return Err(e); } Ok(Some(old)) @@ -562,37 +622,37 @@ fn unhook_route( fn replace_route_targets( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, targets: Vec, ) -> DpdResult<()> { - debug!(ops.log(), "replacing targets for {subnet} with: {targets:?}"); + debug!(ops.log(), "replacing targets for {key} with: {targets:?}"); // Delete-route path: no classification needed; unhook and free. if targets.is_empty() { - let old_entry = unhook_route(ops, route_data, subnet)?; + let old_entry = unhook_route(ops, route_data, key)?; return match old_entry { - Some(entry) => cleanup_route(ops, route_data, subnet, false, entry), + Some(entry) => cleanup_route(ops, route_data, key, entry), None => Ok(()), }; } // Classify against the current in-core entry *before* unhooking so that // a NoOp replace leaves the dataplane untouched (no LPM miss window). - match classify_update(route_data.get(subnet), &targets) { + match classify_update(route_data.get(key), &targets) { RouteTargetUpdate::NoOp => Ok(()), RouteTargetUpdate::ShrinkInPlace { removed } => { - let old = unhook_route(ops, route_data, subnet)? + let old = unhook_route(ops, route_data, key)? .expect("subset removal requires existing route"); // shrink_in_place reconstructs the in-core target vec from the // compacted ASIC layout rather than caller-supplied order (see // its body for why), so the caller's vec carries no useful // information past this point. drop(targets); - shrink_in_place(ops, route_data, subnet, old, removed) + shrink_in_place(ops, route_data, key, old, removed) } RouteTargetUpdate::Alloc => { - let old_entry = unhook_route(ops, route_data, subnet)?; - alloc_then_swap(ops, route_data, subnet, old_entry, targets) + let old_entry = unhook_route(ops, route_data, key)?; + alloc_then_swap(ops, route_data, key, old_entry, targets) } } } @@ -605,12 +665,12 @@ fn replace_route_targets( fn alloc_then_swap( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, old_entry: Option, targets: Vec, ) -> DpdResult<()> { let slots = targets.len() as u8; - let is_ipv4 = subnet.is_ipv4(); + let is_ipv4 = key.subnet.is_ipv4(); let mut new_entry = match route_data.freemap_mut(is_ipv4).alloc(slots) { Ok(index) => RouteEntry { is_ipv4, @@ -625,7 +685,7 @@ fn alloc_then_swap( ); // Restore the old route_index + in-core entry (or no-op if // there was no old entry). - let _ = finalize_route(ops, route_data, subnet, old_entry); + let _ = finalize_route(ops, route_data, key, old_entry); return Err(e); } }; @@ -634,10 +694,10 @@ fn alloc_then_swap( let mut idx = new_entry.index; for target in targets { - if let Err(e) = ops.add_target(subnet, idx, &target) { + if let Err(e) = ops.add_target(key.subnet, idx, &target) { debug!(ops.log(), "failed to insert {target:?} into route table"); - let _ = cleanup_route(ops, route_data, subnet, false, new_entry); - let _ = finalize_route(ops, route_data, subnet, old_entry); + let _ = cleanup_route(ops, route_data, key, new_entry); + let _ = finalize_route(ops, route_data, key, old_entry); return Err(e); } idx += 1; @@ -645,12 +705,12 @@ fn alloc_then_swap( } // Insert the new subnet->index mapping - match finalize_route(ops, route_data, subnet, Some(new_entry.clone())) { + match finalize_route(ops, route_data, key, Some(new_entry.clone())) { Ok(()) => { // Finally free all of the table space for the original set of // targets if let Some(entry) = old_entry { - let _ = cleanup_route(ops, route_data, subnet, false, entry); + let _ = cleanup_route(ops, route_data, key, entry); } Ok(()) } @@ -659,8 +719,8 @@ fn alloc_then_swap( // We failed to point at the new set of targets. Free all of the // new data and update the route_index table to point at the // original set of targets. - let _ = cleanup_route(ops, route_data, subnet, false, new_entry); - let _ = finalize_route(ops, route_data, subnet, old_entry); + let _ = cleanup_route(ops, route_data, key, new_entry); + let _ = finalize_route(ops, route_data, key, old_entry); Err(e) } } @@ -697,7 +757,7 @@ fn alloc_then_swap( fn shrink_in_place( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, old: RouteEntry, removed: Vec, ) -> DpdResult<()> { @@ -724,29 +784,25 @@ fn shrink_in_place( .as_ref() .expect("tail slot is live at this point") .clone(); - if let Err(e) = ops.delete_target(subnet, base + removed_idx) { + if let Err(e) = ops.delete_target(key.subnet, base + removed_idx) { warn!( ops.log(), "shrink-in-place compact delete failed at slot {}: {e:?}", base + removed_idx ); - restore_after_shrink_failure( - ops, route_data, subnet, &live, old, - ); + restore_after_shrink_failure(ops, route_data, key, &live, old); return Err(e); } live[removed_idx as usize] = None; if let Err(e) = - ops.add_target(subnet, base + removed_idx, &tail_contents) + ops.add_target(key.subnet, base + removed_idx, &tail_contents) { warn!( ops.log(), "shrink-in-place compact add failed at slot {}: {e:?}", base + removed_idx ); - restore_after_shrink_failure( - ops, route_data, subnet, &live, old, - ); + restore_after_shrink_failure(ops, route_data, key, &live, old); return Err(e); } live[removed_idx as usize] = Some(tail_contents); @@ -755,12 +811,12 @@ fn shrink_in_place( } // Step 2: install the route_index pointing at the compacted range. - if let Err(e) = ops.add_index(subnet, base, new_n) { + if let Err(e) = ops.add_index(key, base, new_n) { warn!( ops.log(), - "shrink-in-place index re-add failed for {subnet}: {e:?}" + "shrink-in-place index re-add failed for {key}: {e:?}" ); - restore_after_shrink_failure(ops, route_data, subnet, &live, old); + restore_after_shrink_failure(ops, route_data, key, &live, old); return Err(e); } @@ -771,7 +827,7 @@ fn shrink_in_place( let release_count = old.slots as u16 - new_n as u16; let mut all_clear = true; for offset in 0..release_count { - if ops.delete_target(subnet, release_base + offset).is_err() { + if ops.delete_target(key.subnet, release_base + offset).is_err() { all_clear = false; } } @@ -780,7 +836,7 @@ fn shrink_in_place( } else { warn!( ops.log(), - "shrink-in-place tail cleanup partially failed for {subnet}; \ + "shrink-in-place tail cleanup partially failed for {key}; \ leaking slots in the FreeMap's accounting" ); // Skipping the free() call leaves the slots claimed in the FreeMap's @@ -803,12 +859,12 @@ fn shrink_in_place( .map(|opt| opt.expect("live[0..new_n) is populated post-compaction")) .collect(); let prev = route_data.insert( - subnet, + key, RouteEntry { is_ipv4, index: base, slots: new_n, targets: compacted }, ); debug_assert!( prev.is_none(), - "shrink_in_place insert for {subnet} replaced an unexpected \ + "shrink_in_place insert for {key} replaced an unexpected \ existing entry" ); Ok(()) @@ -822,16 +878,16 @@ fn shrink_in_place( fn restore_after_shrink_failure( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, live: &[Option], old: RouteEntry, ) { - if rollback_shrink(ops, subnet, &old, live) { - route_data.insert(subnet, old); + if rollback_shrink(ops, key, &old, live) { + route_data.insert(key, old); } else { error!( ops.log(), - "shrink-in-place rollback failed for {subnet}; in-core entry \ + "shrink-in-place rollback failed for {key}; in-core entry \ stays cleared and ASIC state may diverge until the next \ control-plane update on this subnet rebuilds it" ); @@ -851,7 +907,7 @@ fn restore_after_shrink_failure( // negligible and the bookkeeping savings on the hot path are worth it. fn rollback_shrink( ops: &dyn RouteTableOps, - subnet: IpNet, + key: RouteKey, old: &RouteEntry, live: &[Option], ) -> bool { @@ -864,20 +920,20 @@ fn rollback_shrink( } let slot = base + i as u16; if slot_live.is_some() - && let Err(e) = ops.delete_target(subnet, slot) + && let Err(e) = ops.delete_target(key.subnet, slot) { error!(ops.log(), "rollback delete failed at slot {slot}: {e:?}"); ok = false; } - if let Err(e) = ops.add_target(subnet, slot, orig) { + if let Err(e) = ops.add_target(key.subnet, slot, orig) { error!(ops.log(), "rollback write failed at slot {slot}: {e:?}"); ok = false; } } - if let Err(e) = ops.add_index(subnet, base, old.slots) { + if let Err(e) = ops.add_index(key, base, old.slots) { error!( ops.log(), - "rollback route_index re-add failed for {subnet}: {e:?}" + "rollback route_index re-add failed for {key}: {e:?}" ); ok = false; } @@ -887,18 +943,18 @@ fn rollback_shrink( fn add_route_locked( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, route: Route, asic_port_id: u16, ) -> DpdResult<()> { - info!(ops.log(), "adding route {subnet} -> {:?}", route.tgt_ip); + info!(ops.log(), "adding route {key} -> {:?}", route.tgt_ip); let max_targets = - if subnet.is_ipv4() { MAX_TARGETS_IPV4 } else { MAX_TARGETS_IPV6 }; + if key.subnet.is_ipv4() { MAX_TARGETS_IPV4 } else { MAX_TARGETS_IPV6 }; // Get the old set of targets that we'll be adding to let mut targets = - route_data.get(subnet).map_or(Vec::new(), |e| e.targets.clone()); + route_data.get(key).map_or(Vec::new(), |e| e.targets.clone()); // Add the new target targets.push(NextHop { asic_port_id, route }); @@ -907,7 +963,7 @@ fn add_route_locked( "exceeded limit of {max_targets} targets for one route" ))) } else { - replace_route_targets(ops, route_data, subnet, targets) + replace_route_targets(ops, route_data, key, targets) } } @@ -915,6 +971,7 @@ fn add_route_locked( // just this single target. async fn add_route( switch: &Switch, + router: RouterUuid, subnet: IpNet, route: Route, ) -> DpdResult<()> { @@ -922,14 +979,15 @@ async fn add_route( switch.link_asic_port_id(route.port_id, route.link_id)?; let mut route_data = switch.routes.lock().await; + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); // Adding the same route multiple times is a harmless no-op - if let Some(entry) = route_data.get(subnet) + if let Some(entry) = route_data.get(key) && entry.targets.iter().any(|hop| hop.route == route) { return Ok(()); } - add_route_locked(switch, &mut route_data, subnet, route, asic_port_id) + add_route_locked(switch, &mut route_data, key, route, asic_port_id) } // Create a new single-path route. @@ -940,6 +998,7 @@ async fn add_route( // is, it is not an error to "replace" a non- existent route. async fn set_route( switch: &Switch, + router: RouterUuid, subnet: IpNet, route: Route, replace: bool, @@ -948,19 +1007,20 @@ async fn set_route( switch.link_asic_port_id(route.port_id, route.link_id)?; let mut route_data = switch.routes.lock().await; - if let Some(entry) = route_data.get(subnet) { + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + if let Some(entry) = route_data.get(key) { // setting the same route multiple times is a harmless no-op if entry.targets.len() == 1 && entry.targets[0].route == route { Ok(()) } else if !replace { Err(DpdError::Exists("route {cidr} already exists".into())) } else { - info!(switch.log, "replacing subnet {subnet}"); + info!(switch.log, "replacing subnet {key}"); let target = vec![NextHop { asic_port_id, route }]; - replace_route_targets(switch, &mut route_data, subnet, target) + replace_route_targets(switch, &mut route_data, key, target) } } else { - add_route_locked(switch, &mut route_data, subnet, route, asic_port_id) + add_route_locked(switch, &mut route_data, key, route, asic_port_id) } } @@ -972,13 +1032,13 @@ async fn set_route( fn delete_route_target_locked( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, route: Route, ) -> DpdResult<()> { - info!(ops.log(), "deleting route {subnet} -> {}", route.tgt_ip); + info!(ops.log(), "deleting route {key} -> {}", route.tgt_ip); // Get set of targets remaining after we remove this entry - let entry = route_data.get(subnet).ok_or({ + let entry = route_data.get(key).ok_or({ debug!(ops.log(), "No such route"); DpdError::Missing("no such route".into()) })?; @@ -992,67 +1052,75 @@ fn delete_route_target_locked( debug!(ops.log(), "target not found"); Err(DpdError::Missing("no such route".into())) } else { - replace_route_targets(ops, route_data, subnet, targets) + replace_route_targets(ops, route_data, key, targets) } } pub async fn add_route_ipv4( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, route: Ipv4Route, ) -> DpdResult<()> { - add_route(switch, IpNet::V4(subnet), route.into()).await + add_route(switch, router, subnet.into(), route.into()).await } pub async fn add_route_ipv4_over_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, route: Ipv6Route, ) -> DpdResult<()> { - add_route(switch, IpNet::V4(subnet), route.into()).await + add_route(switch, router, subnet.into(), route.into()).await } pub async fn add_route_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv6Net, route: Ipv6Route, ) -> DpdResult<()> { - add_route(switch, IpNet::V6(subnet), route.into()).await + add_route(switch, router, subnet.into(), route.into()).await } pub async fn set_route_ipv4( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, route: Ipv4Route, replace: bool, ) -> DpdResult<()> { - set_route(switch, IpNet::V4(subnet), route.into(), replace).await + set_route(switch, router, subnet.into(), route.into(), replace).await } pub async fn set_route_ipv4_over_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, route: Ipv6Route, replace: bool, ) -> DpdResult<()> { - set_route(switch, IpNet::V4(subnet), route.into(), replace).await + set_route(switch, router, subnet.into(), route.into(), replace).await } pub async fn set_route_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv6Net, route: Ipv6Route, replace: bool, ) -> DpdResult<()> { - set_route(switch, IpNet::V6(subnet), route.into(), replace).await + set_route(switch, router, subnet.into(), route.into(), replace).await } pub async fn get_route_ipv4( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, ) -> DpdResult> { let route_data = switch.routes.lock().await; - match route_data.get(IpNet::V4(subnet)) { + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + match route_data.get(key) { None => Err(DpdError::Missing("no such route".into())), Some(entry) => { Ok(entry.targets.iter().map(|t| (&t.route).into()).collect()) @@ -1062,10 +1130,12 @@ pub async fn get_route_ipv4( pub async fn get_route_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv6Net, ) -> DpdResult> { let route_data = switch.routes.lock().await; - match route_data.get(IpNet::V6(subnet)) { + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + match route_data.get(key) { None => Err(DpdError::Missing("no such route".into())), Some(entry) => { Ok(entry.targets.iter().map(|t| (&t.route).into()).collect()) @@ -1073,43 +1143,58 @@ pub async fn get_route_ipv6( } } +// Remove a route from the dataplane, then forget it. +// +// The in-core entry is kept for a retry only if deleting the on-chip +// `route_index` fails, since that is what takes the route off the +// dataplane. An index that is already missing counts as deleted. If +// `cleanup_route` then fails to clear a target, the route's slots are +// leaked rather than freed. fn delete_route_locked( ops: &dyn RouteTableOps, route_data: &mut RouteData, - subnet: IpNet, + key: RouteKey, ) -> DpdResult<()> { - // Get set of targets remaining after we remove this entry let entry = route_data - .remove(subnet) + .get(key) + .cloned() .ok_or(DpdError::Missing("no such route".into()))?; - cleanup_route(ops, route_data, subnet, true, entry) + match ops.delete_index(key) { + Ok(()) | Err(DpdError::Switch(AsicError::Missing(_))) => {} + Err(e) => return Err(e), + } + route_data.remove(key); + cleanup_route(ops, route_data, key, entry) } // Delete a route and all of its targets pub async fn delete_route_ipv4( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, ) -> DpdResult<()> { let mut route_data = switch.routes.lock().await; - - delete_route_locked(switch, &mut route_data, IpNet::V4(subnet)) + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + delete_route_locked(switch, &mut route_data, key) } // Delete a route and all of its targets pub async fn delete_route_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv6Net, ) -> DpdResult<()> { let mut route_data = switch.routes.lock().await; - - delete_route_locked(switch, &mut route_data, IpNet::V6(subnet)) + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + delete_route_locked(switch, &mut route_data, key) } // Delete a specific target from a route, removing the route if this is the last // target. pub async fn delete_route_target_ipv4( switch: &Switch, + router: RouterUuid, subnet: Ipv4Net, port_id: PortId, link_id: LinkId, @@ -1119,18 +1204,15 @@ pub async fn delete_route_target_ipv4( Route { tag: String::new(), port_id, link_id, tgt_ip, vlan_id: None }; let mut route_data = switch.routes.lock().await; - delete_route_target_locked( - switch, - &mut route_data, - IpNet::V4(subnet), - route, - ) + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + delete_route_target_locked(switch, &mut route_data, key, route) } // Delete a specific target from a route, removing the route if this is the last // target. pub async fn delete_route_target_ipv6( switch: &Switch, + router: RouterUuid, subnet: Ipv6Net, port_id: PortId, link_id: LinkId, @@ -1145,27 +1227,28 @@ pub async fn delete_route_target_ipv6( }; let mut route_data = switch.routes.lock().await; - delete_route_target_locked( - switch, - &mut route_data, - IpNet::V6(subnet), - route, - ) + let key = RouteKey::new(route_data.routers.table_id(router)?, subnet); + delete_route_target_locked(switch, &mut route_data, key, route) } pub async fn get_range_ipv4( switch: &Switch, + router: RouterUuid, last: Option, max: u32, ) -> DpdResult> { let route_data = switch.routes.lock().await; + let table_id = route_data.routers.table_id(router)?; + let Some(table) = route_data.tables.get(&table_id) else { + return Ok(Vec::new()); + }; let lower = match last { None => Bound::Unbounded, Some(last) => Bound::Excluded(IpNet::V4(last)), }; let mut routes = Vec::new(); - for (subnet, target_list) in route_data + for (subnet, target_list) in table .v4 .range((lower, Bound::Unbounded)) .take(usize::try_from(max).expect("invalid usize")) @@ -1188,17 +1271,22 @@ pub async fn get_range_ipv4( pub async fn get_range_ipv6( switch: &Switch, + router: RouterUuid, last: Option, max: u32, ) -> DpdResult> { let route_data = switch.routes.lock().await; + let table_id = route_data.routers.table_id(router)?; + let Some(table) = route_data.tables.get(&table_id) else { + return Ok(Vec::new()); + }; let lower = match last { None => Bound::Unbounded, Some(last) => Bound::Excluded(IpNet::V6(last)), }; let mut routes = Vec::new(); - for (subnet, target_list) in route_data + for (subnet, target_list) in table .v6 .range((lower, Bound::Unbounded)) .take(usize::try_from(max).expect("invalid usize")) @@ -1230,22 +1318,25 @@ async fn reset_tag(switch: &Switch, tag: &str, ipv4: bool) { // perform any updates after scanning everything to avoid updating the // route_data BTreeMap while we're iterating over it. let mut to_replace = BTreeMap::new(); - let data = if ipv4 { &route_data.v4 } else { &route_data.v6 }; - for (subnet, entry) in data { - let new_targets: Vec = entry - .targets - .iter() - .filter(|t| t.route.tag != tag) - .cloned() - .collect(); - if new_targets.len() != entry.targets.len() { - to_replace.insert(*subnet, new_targets); + for (table_id, table) in route_data.tables.iter() { + let data = if ipv4 { &table.v4 } else { &table.v6 }; + for (subnet, entry) in data { + let new_targets: Vec = entry + .targets + .iter() + .filter(|t| t.route.tag != tag) + .cloned() + .collect(); + if new_targets.len() != entry.targets.len() { + to_replace + .insert(RouteKey::new(*table_id, *subnet), new_targets); + } } } - for (subnet, targets) in to_replace { - debug!(switch.log, "new subnets for {subnet}: {targets:?}"); - let _ = replace_route_targets(switch, &mut route_data, subnet, targets); + for (key, targets) in to_replace { + debug!(switch.log, "new subnets for {key}: {targets:?}"); + let _ = replace_route_targets(switch, &mut route_data, key, targets); } } @@ -1257,23 +1348,29 @@ pub async fn reset_ipv6_tag(switch: &Switch, tag: &str) { reset_tag(switch, tag, false).await } +/// Delete every route in routing table `table_id`. +pub fn delete_table( + switch: &Switch, + route_data: &mut RouteData, + table_id: RoutingTableId, +) -> DpdResult<()> { + route_data.delete_table(switch, table_id) +} + pub async fn reset(switch: &Switch) -> DpdResult<()> { let mut route_data = switch.routes.lock().await; - route_data.v4 = BTreeMap::new(); + route_data.tables.clear(); route_data.v4_freemap.reset(); - table::route_ipv4::reset(switch)?; - - route_data.v6 = BTreeMap::new(); route_data.v6_freemap.reset(); + table::route_ipv4::reset(switch)?; table::route_ipv6::reset(switch)?; - Ok(()) } pub fn init(log: &slog::Logger) -> RouteData { RouteData { - v4: BTreeMap::new(), - v6: BTreeMap::new(), + routers: Routers::default(), + tables: BTreeMap::new(), v4_freemap: freemap::FreeMap::new(log, "route_ipv4"), v6_freemap: freemap::FreeMap::new(log, "route_ipv6"), } @@ -1302,6 +1399,10 @@ mod tests { common::ports::RearPort::new(0).unwrap().into() } + fn default_router_key(subnet: impl Into) -> RouteKey { + RouteKey::new(RoutingTableId::default(), subnet) + } + fn make_route(tgt_ip: IpAddr) -> Route { Route { tag: "test".into(), @@ -1312,8 +1413,8 @@ mod tests { } } - use aal::AsicError; - use std::cell::Cell; + use std::cell::{Cell, RefCell}; + use std::collections::BTreeSet; /// Total size to use for the per-family FreeMap in tests that exercise /// "table full" / "table fragmented" behavior. Small enough to keep @@ -1331,9 +1432,11 @@ mod tests { } /// In-process implementation of `RouteTableOps` used by the route tests. - /// Every on-chip operation is a no-op (returning `Ok`) unless the - /// corresponding failpoint has been armed via [`TestOps::arm`], in which - /// case the configured call returns a synthetic `Switch(SdeError)` and + /// It records which entries are on the fake switch: adding an entry that + /// is already there returns `Exists`, and deleting one that isn't + /// returns `Missing`, as the ASIC does. If the failpoint for an + /// operation has been armed via [`TestOps::arm`], the configured call + /// instead returns a synthetic `Switch(SdeError)`, changes nothing, and /// the failpoint disarms. This lets the route state machine run /// without any ASIC or P4-runtime presence. struct TestOps { @@ -1342,6 +1445,28 @@ mod tests { del_target_fail: Cell>, add_index_fail: Cell>, del_index_fail: Cell>, + /// Installed `route_target` slots, by (is_ipv4, slot). + targets: RefCell>, + indexes: RefCell>, + } + + fn add_entry(set: &RefCell>, key: T) -> DpdResult<()> { + if set.borrow_mut().insert(key) { + Ok(()) + } else { + Err(DpdError::Switch(AsicError::Exists)) + } + } + + fn delete_entry( + set: &RefCell>, + key: &T, + ) -> DpdResult<()> { + if set.borrow_mut().remove(key) { + Ok(()) + } else { + Err(DpdError::Switch(AsicError::Missing("not installed".into()))) + } } impl TestOps { @@ -1358,6 +1483,8 @@ mod tests { del_target_fail: Cell::new(None), add_index_fail: Cell::new(None), del_index_fail: Cell::new(None), + targets: RefCell::default(), + indexes: RefCell::default(), } } @@ -1403,28 +1530,32 @@ mod tests { fn add_target( &self, - _subnet: IpNet, - _idx: u16, + subnet: IpNet, + idx: u16, _target: &NextHop, ) -> DpdResult<()> { - self.check(Op::AddTarget) + self.check(Op::AddTarget)?; + add_entry(&self.targets, (subnet.is_ipv4(), idx)) } - fn delete_target(&self, _subnet: IpNet, _idx: u16) -> DpdResult<()> { - self.check(Op::DelTarget) + fn delete_target(&self, subnet: IpNet, idx: u16) -> DpdResult<()> { + self.check(Op::DelTarget)?; + delete_entry(&self.targets, &(subnet.is_ipv4(), idx)) } fn add_index( &self, - _subnet: IpNet, + key: RouteKey, _index: u16, _slots: u8, ) -> DpdResult<()> { - self.check(Op::AddIndex) + self.check(Op::AddIndex)?; + add_entry(&self.indexes, key) } - fn delete_index(&self, _subnet: IpNet) -> DpdResult<()> { - self.check(Op::DelIndex) + fn delete_index(&self, key: RouteKey) -> DpdResult<()> { + self.check(Op::DelIndex)?; + delete_entry(&self.indexes, &key) } } @@ -1444,7 +1575,7 @@ mod tests { fn install_victim( ops: &dyn RouteTableOps, rd: &mut RouteData, - victim: IpNet, + victim: RouteKey, targets: impl IntoIterator, ) { for tgt in targets { @@ -1468,7 +1599,7 @@ mod tests { match add_route_locked( ops, rd, - cidr, + default_router_key(cidr), make_route(filler), FAKE_ASIC_PORT, ) { @@ -1494,7 +1625,8 @@ mod tests { fillers.len(), ); for f in fillers.iter().step_by(2) { - delete_route_locked(ops, rd, *f).expect("delete filler"); + delete_route_locked(ops, rd, default_router_key(*f)) + .expect("delete filler"); } } @@ -1504,7 +1636,7 @@ mod tests { /// "shared tgt_ip with one survivor" multi-target scenario. struct FamilyFixtures { is_ipv4: bool, - victim: IpNet, + victim: RouteKey, targets: [IpAddr; 4], filler: IpAddr, filler_cidr: fn(u32) -> IpNet, @@ -1513,7 +1645,9 @@ mod tests { fn fixtures_v6() -> FamilyFixtures { FamilyFixtures { is_ipv4: false, - victim: "3fff:dead::/64".parse::().unwrap().into(), + victim: default_router_key( + "3fff:dead::/64".parse::().unwrap(), + ), targets: [ "2001:db8::55:1".parse::().unwrap().into(), "2001:db8::55:2".parse::().unwrap().into(), @@ -1528,7 +1662,9 @@ mod tests { fn fixtures_v4() -> FamilyFixtures { FamilyFixtures { is_ipv4: true, - victim: "172.16.0.0/32".parse::().unwrap().into(), + victim: default_router_key( + "172.16.0.0/32".parse::().unwrap(), + ), targets: [ Ipv4Addr::new(10, 0, 0, 1).into(), Ipv4Addr::new(10, 0, 0, 2).into(), @@ -1570,13 +1706,13 @@ mod tests { /// should treat it as a terminal assertion. fn assert_post_shrink( rd: &mut RouteData, - subnet: IpNet, + key: RouteKey, is_ipv4: bool, expected: &[IpAddr], freed_slots: u16, ) { use std::collections::BTreeSet; - let entry = rd.get(subnet).expect("victim must still exist"); + let entry = rd.get(key).expect("victim must still exist"); let observed: BTreeSet = entry.targets.iter().map(|t| t.route.tgt_ip).collect(); let expected_set: BTreeSet = expected.iter().copied().collect(); @@ -1852,6 +1988,113 @@ mod tests { identity_replace_is_noop_scenario(fixtures_v4()); } + /// Deleting a routing table removes its routes from the switch and from + /// soft state, and leaves other tables' entries for the same prefixes + /// alone. + #[test] + fn delete_table_leaves_other_tables() { + let (ops, mut rd) = make_test_ctx(); + let doomed = RoutingTableId::from(1); + let other = RoutingTableId::from(2); + let (f4, f6) = (fixtures_v4(), fixtures_v6()); + for table_id in [RoutingTableId::default(), doomed, other] { + install_victim( + &ops, + &mut rd, + RouteKey::new(table_id, f4.victim.subnet), + [f4.targets[0]], + ); + install_victim( + &ops, + &mut rd, + RouteKey::new(table_id, f6.victim.subnet), + [f6.targets[0]], + ); + } + + rd.delete_table(&ops, doomed).unwrap(); + assert!(!rd.tables.contains_key(&doomed)); + for subnet in [f4.victim.subnet, f6.victim.subnet] { + let gone = RouteKey::new(doomed, subnet); + assert!(!ops.indexes.borrow().contains(&gone)); + for table_id in [RoutingTableId::default(), other] { + let key = RouteKey::new(table_id, subnet); + assert!(rd.get(key).is_some()); + assert!(ops.indexes.borrow().contains(&key)); + } + } + + // Deleting it again does nothing. + rd.delete_table(&ops, doomed).unwrap(); + } + + /// A failed index delete keeps the route, and a retry removes it. + #[test] + fn delete_table_keeps_route_when_index_delete_fails() { + let (ops, mut rd) = make_test_ctx(); + let table_id = RoutingTableId::from(1); + let f = fixtures_v4(); + let key = RouteKey::new(table_id, f.victim.subnet); + install_victim(&ops, &mut rd, key, [f.targets[0]]); + + ops.arm(Op::DelIndex, 0); + assert!(rd.delete_table(&ops, table_id).is_err()); + assert!(rd.get(key).is_some(), "route must be kept for a retry"); + + rd.delete_table(&ops, table_id).expect("retry must succeed"); + assert!(!rd.tables.contains_key(&table_id)); + assert!(ops.indexes.borrow().is_empty()); + assert!(ops.targets.borrow().is_empty()); + } + + /// A failed target delete still forgets the route, and the slot whose + /// target is still installed is not handed out again. + #[test] + fn delete_table_forgets_route_when_target_delete_fails() { + let (ops, mut rd) = make_test_ctx(); + let table_id = RoutingTableId::from(1); + let f = fixtures_v4(); + let key = RouteKey::new(table_id, f.victim.subnet); + install_victim(&ops, &mut rd, key, [f.targets[0], f.targets[1]]); + let entry = rd.get(key).unwrap().clone(); + + // Fail the second target delete + ops.arm(Op::DelTarget, 1); + rd.delete_table(&ops, table_id).expect("delete must succeed"); + assert!(!rd.tables.contains_key(&table_id)); + assert!(rd.get(key).is_none()); + assert!(!ops.indexes.borrow().contains(&key)); + + // The second target's delete failed, so its slot must not be handed + // out again. + let still_installed = entry.index + 1; + let idx = rd.freemap_mut(true).alloc(entry.slots).unwrap(); + assert!(!(idx..idx + entry.slots as u16).contains(&still_installed)); + } + + /// After a delete whose target cleanup fails, the route is gone, so adding + /// it again installs a fresh index. + #[test] + fn readd_after_failed_target_delete_installs_route() { + let (ops, mut rd) = make_test_ctx(); + let f = fixtures_v4(); + install_victim(&ops, &mut rd, f.victim, [f.targets[0]]); + + ops.arm(Op::DelTarget, 0); + let _ = delete_route_locked(&ops, &mut rd, f.victim); + + add_route_locked( + &ops, + &mut rd, + f.victim, + make_route(f.targets[0]), + FAKE_ASIC_PORT, + ) + .expect("re-add must succeed"); + assert!(rd.get(f.victim).is_some()); + assert!(ops.indexes.borrow().contains(&f.victim)); + } + /// A same-length, non-subset replace (one target swapped for another) /// must take the alloc-then-swap path: the new set isn't ⊆ old, so /// shrink-in-place doesn't apply. Verifies the dispatcher routes diff --git a/dpd/src/router.rs b/dpd/src/router.rs new file mode 100644 index 00000000..afa67d6c --- /dev/null +++ b/dpd/src/router.rs @@ -0,0 +1,218 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use std::collections::BTreeMap; +use std::net::Ipv6Addr; + +use common::ports::Ipv6Entry; +use uuid::Uuid; + +use dpd_types::route::DEFAULT_ROUTER; + +use crate::route::{self, RouteData}; +use crate::types::{DpdError, DpdResult}; +use crate::{Switch, loopback}; + +/// A router's uuid, as used in the API. The nil uuid (`DEFAULT_ROUTER`, also +/// `RouterUuid::default()`) names table 0. +#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] +pub struct RouterUuid(Uuid); + +impl RouterUuid { + pub fn is_default(&self) -> bool { + self.0 == DEFAULT_ROUTER + } +} + +impl From for RouterUuid { + fn from(uuid: Uuid) -> Self { + RouterUuid(uuid) + } +} + +impl From for Uuid { + fn from(router: RouterUuid) -> Self { + router.0 + } +} + +impl std::fmt::Display for RouterUuid { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// The number of a routing table on the switch. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct RoutingTableId(u8); + +/// The routing table for underlay routes and the `DEFAULT_ROUTER`. +impl Default for RoutingTableId { + fn default() -> Self { + RoutingTableId(0) + } +} + +impl From for RoutingTableId { + fn from(table_id: u8) -> Self { + RoutingTableId(table_id) + } +} + +impl From for u8 { + fn from(table_id: RoutingTableId) -> Self { + table_id.0 + } +} + +impl std::fmt::Display for RoutingTableId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A logical router: its endpoint and its routing table id. +#[derive(Clone, Copy)] +struct Router { + endpoint: Ipv6Addr, + table_id: RoutingTableId, +} + +/// The logical routers keyed by their control-plane uuid. +#[derive(Default)] +pub struct Routers(BTreeMap); + +impl Routers { + /// The routing table used by `router_uuid`. + pub fn table_id( + &self, + router_uuid: RouterUuid, + ) -> DpdResult { + if router_uuid.is_default() { + return Ok(RoutingTableId::default()); + } + self.0.get(&router_uuid).map(|r| r.table_id).ok_or_else(|| { + DpdError::Missing(format!("no such router {router_uuid}")) + }) + } + + /// Checks if endpoint address is taken. + pub fn check_not_endpoint(&self, addr: Ipv6Addr) -> DpdResult<()> { + match self.0.iter().find(|(_, r)| r.endpoint == addr) { + Some((router, _)) => Err(DpdError::Exists(format!( + "{addr} is the endpoint of router {router}" + ))), + None => Ok(()), + } + } + + /// Lists endpoints by their uuids. + pub fn list(&self) -> BTreeMap { + self.0 + .iter() + .map(|(router, r)| ((*router).into(), r.endpoint)) + .collect() + } + + /// Returns the endpoint of `router_uuid`. + pub fn endpoint(&self, router_uuid: RouterUuid) -> DpdResult { + self.0.get(&router_uuid).map(|r| r.endpoint).ok_or_else(|| { + DpdError::Missing(format!("no such router {router_uuid}")) + }) + } + + /// Create `router`, claiming `endpoint` as a loopback for the router's + /// routing table. + pub fn create( + &mut self, + switch: &Switch, + router: RouterUuid, + endpoint: Ipv6Addr, + ) -> DpdResult<()> { + if router.is_default() { + return Err(DpdError::Invalid(format!( + "router {router} names table 0, which can't be created" + ))); + } + if let Some(r) = self.0.get(&router) { + if r.endpoint == endpoint { + // A failed delete may have removed the endpoint; reinstall it. + let entry = + Ipv6Entry { tag: router.to_string(), addr: endpoint }; + return match loopback::add_loopback_ipv6( + switch, &entry, r.table_id, + ) { + Ok(()) | Err(DpdError::Exists(_)) => Ok(()), + Err(e) => Err(e), + }; + } + return Err(DpdError::Exists(format!( + "router {router} already has endpoint {}", + r.endpoint + ))); + } + // An endpoint needs to map to a single router. + if self.0.values().any(|r| r.endpoint == endpoint) { + return Err(DpdError::Exists(format!( + "{endpoint} is already taken" + ))); + } + let table_id = (1..=u8::MAX) + .map(RoutingTableId::from) + .find(|table_id| self.0.values().all(|r| r.table_id != *table_id)) + .ok_or_else(|| { + DpdError::TableFull("no routing tables available".into()) + })?; + let entry = Ipv6Entry { tag: router.to_string(), addr: endpoint }; + loopback::add_loopback_ipv6(switch, &entry, table_id)?; + self.0.insert(router, Router { endpoint, table_id }); + Ok(()) + } +} + +/// Delete `router` along with its endpoint and routes. +/// +/// The router is forgotten only once everything under it has been removed, +/// so its table is never given to another router while entries for it +/// remain. If a removal fails, the router still exists; the delete can be +/// retried or a PUT with the same endpoint can restore it. +pub fn delete( + switch: &Switch, + route_data: &mut RouteData, + router: RouterUuid, +) -> DpdResult<()> { + if router.is_default() { + return Err(DpdError::Invalid(format!( + "router {router} names table 0, which can't be deleted" + ))); + } + remove(switch, route_data, router) +} + +/// Delete every router. +pub fn reset(switch: &Switch, route_data: &mut RouteData) -> DpdResult<()> { + let all: Vec = route_data.routers.0.keys().copied().collect(); + for router in all { + remove(switch, route_data, router)?; + } + Ok(()) +} + +fn remove( + switch: &Switch, + route_data: &mut RouteData, + router: RouterUuid, +) -> DpdResult<()> { + let Some(&Router { endpoint, table_id }) = + route_data.routers.0.get(&router) + else { + return Ok(()); + }; + loopback::delete_loopback_ipv6(switch, &endpoint)?; + route::delete_table(switch, route_data, table_id)?; + route_data.routers.0.remove(&router); + Ok(()) +} diff --git a/dpd/src/table/port_ip.rs b/dpd/src/table/port_ip.rs index 322493a5..f87b1838 100644 --- a/dpd/src/table/port_ip.rs +++ b/dpd/src/table/port_ip.rs @@ -9,6 +9,7 @@ use std::convert::TryInto; use std::net::Ipv4Addr; use std::net::Ipv6Addr; +use crate::router::RoutingTableId; use slog::error; use slog::info; @@ -45,7 +46,7 @@ enum ActionV4 { #[derive(ActionParse)] enum ActionV6 { #[action_xlate(name = "claimv6")] - ClaimIpv6, + ClaimIpv6 { routing_table_id: u8 }, #[action_xlate(name = "dropv6")] DropIpv6, } @@ -119,16 +120,28 @@ pub fn loopback_ipv4_delete(s: &Switch, ipv4: Ipv4Addr) -> DpdResult<()> { }) } -pub fn loopback_ipv6_add(s: &Switch, ipv6: Ipv6Addr) -> DpdResult<()> { +pub fn loopback_ipv6_add( + s: &Switch, + ipv6: Ipv6Addr, + table_id: RoutingTableId, +) -> DpdResult<()> { let claim_key = Ipv6MatchKey { dst_addr: ipv6, in_port: MatchMask { val: 0u16.into(), mask: 0u16.into() }, }; - s.table_entry_add(TableType::PortAddrIpv6, &claim_key, &ActionV6::ClaimIpv6) - .map(|_| info!(s.log, "added ipv6 loopback"; "addr" => %ipv6)) + + let action = ActionV6::ClaimIpv6 { routing_table_id: table_id.into() }; + + s.table_entry_add(TableType::PortAddrIpv6, &claim_key, &action) + .map(|_| { + info!(s.log, "added ipv6 loopback"; + "addr" => %ipv6, + "routing_table_id" => %table_id) + }) .inspect_err(|e| { error!(s.log, "failed to add ipv6 loopback"; "addr" => %ipv6, + "routing_table_id" => %table_id, "error" => %e); }) } @@ -252,7 +265,9 @@ fn ipv6_add_work(s: &Switch, port: u16, ipv6: Ipv6Addr) -> DpdResult<()> { s.table_entry_add( TableType::PortAddrIpv6, &claim_key, - &ActionV6::ClaimIpv6, + &ActionV6::ClaimIpv6 { + routing_table_id: RoutingTableId::default().into(), + }, )?; s.table_entry_add(TableType::PortAddrIpv6, &drop_key, &ActionV6::DropIpv6) .inspect_err(|_| { @@ -292,7 +307,9 @@ fn ipv6_delete_work(s: &Switch, port: u16, ipv6: Ipv6Addr) -> DpdResult<()> { s.table_entry_add( TableType::PortAddrIpv6, &claim_key, - &ActionV6::ClaimIpv6, + &ActionV6::ClaimIpv6 { + routing_table_id: RoutingTableId::default().into(), + }, ) }, ); diff --git a/dpd/src/table/route_ipv4.rs b/dpd/src/table/route_ipv4.rs index e5b9d79e..7a5c7dd9 100644 --- a/dpd/src/table/route_ipv4.rs +++ b/dpd/src/table/route_ipv4.rs @@ -4,6 +4,7 @@ // // Copyright 2026 Oxide Computer Company +use crate::router::RoutingTableId; use dpd_types::table; use std::convert::TryInto; use std::net::Ipv4Addr; @@ -41,6 +42,8 @@ enum RouteAction { // Used to identify entries in the route->index table #[derive(MatchParse, Hash, Debug)] struct RouteKey { + #[match_xlate(type = "value")] + routing_table_id: u8, #[match_xlate(type = "lpm")] dst_addr: Ipv4Net, } @@ -55,17 +58,20 @@ enum IndexAction { /// Add an entry to the route->index table pub fn add_route_index( s: &Switch, + table_id: RoutingTableId, cidr: &Ipv4Net, idx: u16, slots: u8, ) -> DpdResult<()> { let action_data = IndexAction::Index { idx, slots }; - let match_key = RouteKey { dst_addr: *cidr }; + let match_key = + RouteKey { routing_table_id: u8::from(table_id), dst_addr: *cidr }; match s.table_entry_add(TableType::RouteIdxIpv4, &match_key, &action_data) { Ok(()) => { info!(s.log, "added ipv4 route index"; + "routing_table_id" => %table_id, "route" => %cidr, "index" => %idx, "slots" => %slots); @@ -73,6 +79,7 @@ pub fn add_route_index( } Err(e) => { error!(s.log, "failed to add ipv4 route index"; + "routing_table_id" => %table_id, "route" => %cidr, "index" => %idx, "slots" => %slots, @@ -83,13 +90,23 @@ pub fn add_route_index( } /// Remove an entry from the route->index table -pub fn delete_route_index(s: &Switch, cidr: &Ipv4Net) -> DpdResult<()> { - let match_key = RouteKey { dst_addr: *cidr }; +pub fn delete_route_index( + s: &Switch, + table_id: RoutingTableId, + cidr: &Ipv4Net, +) -> DpdResult<()> { + let match_key = + RouteKey { routing_table_id: u8::from(table_id), dst_addr: *cidr }; s.table_entry_del(TableType::RouteIdxIpv4, &match_key) - .map(|_| info!(s.log, "deleted ipv4 index"; "route" => %cidr)) + .map(|_| { + info!(s.log, "deleted ipv4 index"; + "routing_table_id" => %table_id, + "route" => %cidr) + }) .map_err(|e| { error!(s.log, "failed to delete ipv4 index"; + "routing_table_id" => %table_id, "route" => %cidr, "error" => %e); e diff --git a/dpd/src/table/route_ipv6.rs b/dpd/src/table/route_ipv6.rs index 81dc2ec3..1196b4bf 100644 --- a/dpd/src/table/route_ipv6.rs +++ b/dpd/src/table/route_ipv6.rs @@ -4,6 +4,7 @@ // // Copyright 2026 Oxide Computer Company +use crate::router::RoutingTableId; use dpd_types::table; use std::convert::TryInto; use std::net::Ipv6Addr; @@ -36,6 +37,8 @@ enum RouteAction { // Used to identify entries in the route->index table #[derive(MatchParse, Hash, Debug)] struct RouteKey { + #[match_xlate(type = "value")] + routing_table_id: u8, #[match_xlate(type = "lpm")] dst_addr: Ipv6Net, } @@ -50,17 +53,20 @@ enum IndexAction { /// Add an entry to the route->index table pub fn add_route_index( s: &Switch, + table_id: RoutingTableId, cidr: &Ipv6Net, idx: u16, slots: u8, ) -> DpdResult<()> { let action_data = IndexAction::Index { idx, slots }; - let match_key = RouteKey { dst_addr: *cidr }; + let match_key = + RouteKey { routing_table_id: u8::from(table_id), dst_addr: *cidr }; match s.table_entry_add(TableType::RouteIdxIpv6, &match_key, &action_data) { Ok(()) => { info!(s.log, "added ipv6 route index"; + "routing_table_id" => %table_id, "route" => %cidr, "index" => %idx, "slots" => %slots); @@ -68,6 +74,7 @@ pub fn add_route_index( } Err(e) => { error!(s.log, "failed to add ipv6 route index"; + "routing_table_id" => %table_id, "route" => %cidr, "index" => %idx, "slots" => %slots, @@ -78,13 +85,23 @@ pub fn add_route_index( } /// Remove an entry from the route->index table -pub fn delete_route_index(s: &Switch, cidr: &Ipv6Net) -> DpdResult<()> { - let match_key = RouteKey { dst_addr: *cidr }; +pub fn delete_route_index( + s: &Switch, + table_id: RoutingTableId, + cidr: &Ipv6Net, +) -> DpdResult<()> { + let match_key = + RouteKey { routing_table_id: u8::from(table_id), dst_addr: *cidr }; s.table_entry_del(TableType::RouteIdxIpv6, &match_key) - .map(|_| info!(s.log, "deleted ipv6 index"; "route" => %cidr)) + .map(|_| { + info!(s.log, "deleted ipv6 index"; + "routing_table_id" => %table_id, + "route" => %cidr) + }) .map_err(|e| { error!(s.log, "failed to delete ipv6 index"; + "routing_table_id" => %table_id, "route" => %cidr, "error" => %e); e diff --git a/openapi/dpd/dpd-13.0.0-5db8bd.json.gitstub b/openapi/dpd/dpd-13.0.0-5db8bd.json.gitstub new file mode 100644 index 00000000..c886f19e --- /dev/null +++ b/openapi/dpd/dpd-13.0.0-5db8bd.json.gitstub @@ -0,0 +1 @@ +2df9101e95757037f2cc4e31e8bf00fd9e79efc7:openapi/dpd/dpd-13.0.0-5db8bd.json diff --git a/openapi/dpd/dpd-13.0.0-5db8bd.json b/openapi/dpd/dpd-14.0.0-adf422.json similarity index 93% rename from openapi/dpd/dpd-13.0.0-5db8bd.json rename to openapi/dpd/dpd-14.0.0-adf422.json index 22694237..cd87ad05 100644 --- a/openapi/dpd/dpd-13.0.0-5db8bd.json +++ b/openapi/dpd/dpd-14.0.0-adf422.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "13.0.0" + "version": "14.0.0" }, "paths": { "/all-settings": { @@ -4829,8 +4829,8 @@ }, "/route/ipv4": { "get": { - "summary": "Fetch the configured IPv4 routes, mapping IPv4 CIDR blocks to the switch port", - "description": "used for sending out that traffic, and optionally a gateway.", + "summary": "Fetch the IPv4 routes configured on the default routing table, mapping", + "description": "IPv4 CIDR blocks to the switch port used for sending out that traffic, and optionally a gateway.", "operationId": "route_ipv4_list", "parameters": [ { @@ -4877,8 +4877,8 @@ } }, "put": { - "summary": "Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6).", - "description": "This call can be used to create a new single-path route or to replace any existing routes with a new single-path route.", + "summary": "Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on", + "description": "the default routing table.\n\nThis call can be used to create a new single-path route or to replace any existing routes with a new single-path route.", "operationId": "route_ipv4_set", "requestBody": { "content": { @@ -4903,8 +4903,8 @@ } }, "post": { - "summary": "Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6).", - "description": "This call can be used to create a new single-path route or to add new targets to a multipath route.", + "summary": "Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on", + "description": "the default routing table.\n\nThis call can be used to create a new single-path route or to add new targets to a multipath route.", "operationId": "route_ipv4_add", "requestBody": { "content": { @@ -4931,7 +4931,8 @@ }, "/route/ipv4/{cidr}": { "get": { - "summary": "Get the configured route for the given IPv4 subnet.", + "summary": "Get the configured route for the given IPv4 subnet on the default routing", + "description": "table.", "operationId": "route_ipv4_get", "parameters": [ { @@ -4968,7 +4969,7 @@ } }, "delete": { - "summary": "Remove all targets for the given subnet", + "summary": "Remove all targets for the given subnet on the default routing table", "operationId": "route_ipv4_delete", "parameters": [ { @@ -4996,7 +4997,8 @@ }, "/route/ipv4/{cidr}/{port_id}/{link_id}/{tgt_ip}": { "delete": { - "summary": "Remove a single target for the given IPv4 subnet (IPv4 or IPv6 next hop)", + "summary": "Remove a single target for the given IPv4 subnet on the default routing", + "description": "table (IPv4 or IPv6 next hop)", "operationId": "route_ipv4_delete_target", "parameters": [ { @@ -5052,8 +5054,8 @@ }, "/route/ipv6": { "get": { - "summary": "Fetch the configured IPv6 routes, mapping IPv6 CIDR blocks to the switch port", - "description": "used for sending out that traffic, and optionally a gateway.", + "summary": "Fetch the IPv6 routes configured on the default routing table, mapping", + "description": "IPv6 CIDR blocks to the switch port used for sending out that traffic, and optionally a gateway.", "operationId": "route_ipv6_list", "parameters": [ { @@ -5100,8 +5102,8 @@ } }, "put": { - "summary": "Route an IPv6 subnet to a link and a nexthop gateway.", - "description": "This call can be used to create a new single-path route or to replace any existing routes with a new single-path route.", + "summary": "Route an IPv6 subnet to a link and a nexthop gateway on the default", + "description": "routing table.\n\nThis call can be used to create a new single-path route or to replace any existing routes with a new single-path route.", "operationId": "route_ipv6_set", "requestBody": { "content": { @@ -5126,8 +5128,8 @@ } }, "post": { - "summary": "Route an IPv6 subnet to a link and a nexthop gateway.", - "description": "This call can be used to create a new single-path route or to add new targets to a multipath route.", + "summary": "Route an IPv6 subnet to a link and a nexthop gateway on the default", + "description": "routing table.\n\nThis call can be used to create a new single-path route or to add new targets to a multipath route.", "operationId": "route_ipv6_add", "requestBody": { "content": { @@ -5154,7 +5156,8 @@ }, "/route/ipv6/{cidr}": { "get": { - "summary": "Get a single IPv6 route, by its IPv6 CIDR block.", + "summary": "Get a single IPv6 route on the default routing table, by its IPv6 CIDR", + "description": "block.", "operationId": "route_ipv6_get", "parameters": [ { @@ -5191,7 +5194,8 @@ } }, "delete": { - "summary": "Remove an IPv6 route, by its IPv6 CIDR block.", + "summary": "Remove an IPv6 route on the default routing table, by its IPv6 CIDR", + "description": "block.", "operationId": "route_ipv6_delete", "parameters": [ { @@ -5219,7 +5223,8 @@ }, "/route/ipv6/{cidr}/{port_id}/{link_id}/{tgt_ip}": { "delete": { - "summary": "Remove a single target for the given IPv6 subnet", + "summary": "Remove a single target for the given IPv6 subnet on the default routing", + "description": "table", "operationId": "route_ipv6_delete_target", "parameters": [ { @@ -5273,6 +5278,716 @@ } } }, + "/router": { + "get": { + "summary": "List the routers with their endpoints.", + "operationId": "router_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_ipv6", + "type": "object", + "additionalProperties": { + "type": "string", + "format": "ipv6" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}": { + "get": { + "summary": "Get a router's endpoint.", + "operationId": "router_get", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "ipv6", + "type": "string", + "format": "ipv6" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Create a router with the given endpoint. An address can be the", + "description": "endpoint of only one router, and a router's endpoint can't be changed. The default routing table's uuid can't be used.", + "operationId": "router_create", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "ipv6", + "type": "string", + "format": "ipv6" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a router, along with its endpoint and routes. The default", + "description": "routing table's uuid can't be used.", + "operationId": "router_delete", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}/route/ipv4": { + "get": { + "summary": "Fetch the IPv4 routes configured on the given router, mapping IPv4", + "description": "CIDR blocks to the switch port used for sending out that traffic, and optionally a gateway.", + "operationId": "router_route_ipv4_list", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv4RoutesResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, + "put": { + "summary": "Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on", + "description": "the given router.\n\nThis call can be used to create a new single-path route or to replace any existing routes with a new single-path route.", + "operationId": "router_route_ipv4_set", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv4RouteUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "summary": "Route an IPv4 subnet to a link and a nexthop gateway (IPv4 or IPv6) on", + "description": "the given router.\n\nThis call can be used to create a new single-path route or to add new targets to a multipath route.", + "operationId": "router_route_ipv4_add", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv4RouteUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}/route/ipv4/{cidr}": { + "get": { + "summary": "Get the configured route for the given IPv4 subnet on the given router.", + "operationId": "router_route_ipv4_get", + "parameters": [ + { + "in": "path", + "name": "cidr", + "description": "The IPv4 subnet in CIDR notation whose route entry is returned.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ipv4Net" + } + }, + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Route", + "type": "array", + "items": { + "$ref": "#/components/schemas/Route" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Remove all targets for the given subnet on the given router.", + "operationId": "router_route_ipv4_delete", + "parameters": [ + { + "in": "path", + "name": "cidr", + "description": "The IPv4 subnet in CIDR notation whose route entry is returned.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ipv4Net" + } + }, + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}/route/ipv4/{cidr}/{port_id}/{link_id}/{tgt_ip}": { + "delete": { + "summary": "Remove a single target for the given IPv4 subnet on the given router", + "description": "(IPv4 or IPv6 next hop).", + "operationId": "router_route_ipv4_delete_target", + "parameters": [ + { + "in": "path", + "name": "cidr", + "description": "The subnet being routed", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ipv4Net" + } + }, + { + "in": "path", + "name": "link_id", + "description": "The link to which packets should be sent", + "required": true, + "schema": { + "$ref": "#/components/schemas/LinkId" + } + }, + { + "in": "path", + "name": "port_id", + "description": "The switch port to which packets should be sent", + "required": true, + "schema": { + "$ref": "#/components/schemas/PortId" + } + }, + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "tgt_ip", + "description": "The next hop in the route (IPv4 or IPv6)", + "required": true, + "schema": { + "type": "string", + "format": "ip" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}/route/ipv6": { + "get": { + "summary": "Fetch the IPv6 routes configured on the given router, mapping IPv6", + "description": "CIDR blocks to the switch port used for sending out that traffic, and optionally a gateway.", + "operationId": "router_route_ipv6_list", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv6RoutesResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, + "put": { + "summary": "Route an IPv6 subnet to a link and a nexthop gateway on the given", + "description": "router.\n\nThis call can be used to create a new single-path route or to replace any existing routes with a new single-path route.", + "operationId": "router_route_ipv6_set", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv6RouteUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "summary": "Route an IPv6 subnet to a link and a nexthop gateway on the given", + "description": "router.\n\nThis call can be used to create a new single-path route or to add new targets to a multipath route.", + "operationId": "router_route_ipv6_add", + "parameters": [ + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ipv6RouteUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}/route/ipv6/{cidr}": { + "get": { + "summary": "Get a single IPv6 route on the given router, by its IPv6 CIDR block.", + "operationId": "router_route_ipv6_get", + "parameters": [ + { + "in": "path", + "name": "cidr", + "description": "The IPv6 subnet in CIDR notation whose route entry is returned.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Ipv6Route", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Route" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Remove an IPv6 route on the given router, by its IPv6 CIDR block.", + "operationId": "router_route_ipv6_delete", + "parameters": [ + { + "in": "path", + "name": "cidr", + "description": "The IPv6 subnet in CIDR notation whose route entry is returned.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/router/{router_id}/route/ipv6/{cidr}/{port_id}/{link_id}/{tgt_ip}": { + "delete": { + "summary": "Remove a single target for the given IPv6 subnet on the given router.", + "operationId": "router_route_ipv6_delete_target", + "parameters": [ + { + "in": "path", + "name": "cidr", + "description": "The subnet being routed", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + { + "in": "path", + "name": "link_id", + "description": "The link to which packets should be sent", + "required": true, + "schema": { + "$ref": "#/components/schemas/LinkId" + } + }, + { + "in": "path", + "name": "port_id", + "description": "The switch port to which packets should be sent", + "required": true, + "schema": { + "$ref": "#/components/schemas/PortId" + } + }, + { + "in": "path", + "name": "router_id", + "description": "The router being addressed.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "tgt_ip", + "description": "The next hop in the IPv6 route", + "required": true, + "schema": { + "type": "string", + "format": "ipv6" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/rpw/nat/gen": { "get": { "summary": "Get NAT generation number", diff --git a/openapi/dpd/dpd-latest.json b/openapi/dpd/dpd-latest.json index d1a5660e..888260c6 120000 --- a/openapi/dpd/dpd-latest.json +++ b/openapi/dpd/dpd-latest.json @@ -1 +1 @@ -dpd-13.0.0-5db8bd.json \ No newline at end of file +dpd-14.0.0-adf422.json \ No newline at end of file diff --git a/swadm/Cargo.toml b/swadm/Cargo.toml index 57ceccdf..02f7d72f 100644 --- a/swadm/Cargo.toml +++ b/swadm/Cargo.toml @@ -24,6 +24,7 @@ reqwest.workspace = true slog.workspace = true tabwriter.workspace = true tokio.workspace = true +uuid.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/swadm/src/main.rs b/swadm/src/main.rs index b5fd1221..de8be1ba 100644 --- a/swadm/src/main.rs +++ b/swadm/src/main.rs @@ -11,6 +11,7 @@ use std::str::FromStr; use anyhow::Context; use clap::{Parser, Subcommand, ValueEnum}; +use uuid::Uuid; use dpd_client::Client; use dpd_client::ClientState; @@ -25,6 +26,7 @@ mod counters; mod link; mod nat; mod route; +mod router; mod snapshot; mod switchport; mod table; @@ -55,6 +57,9 @@ enum Commands { cmd: arp::Arp, }, Route { + /// router to operate on (default: the default routing table) + #[clap(long, global = true)] + router_id: Option, #[command(subcommand)] cmd: route::Route, }, @@ -62,6 +67,10 @@ enum Commands { #[command(subcommand)] cmd: addr::Addr, }, + Router { + #[command(subcommand)] + cmd: router::Router, + }, Nat { #[command(subcommand)] cmd: nat::Nat, @@ -237,8 +246,13 @@ async fn main_impl() -> anyhow::Result<()> { match opts.cmd { Commands::DpdBuildInfo => build_info(&client).await, Commands::Arp { cmd: a } => arp::arp_cmd(&client, a).await, - Commands::Route { cmd: r } => route::route_cmd(&client, r).await, + Commands::Route { router_id, cmd: r } => { + // The nil uuid names the default routing table. + let router_id = router_id.unwrap_or(Uuid::nil()); + route::route_cmd(&client, router_id, r).await + } Commands::Addr { cmd: p } => addr::addr_cmd(&client, p).await, + Commands::Router { cmd } => router::router_cmd(&client, cmd).await, Commands::AttachedSubnet { cmd: p } => { attached::attsub_cmd(&client, p).await } diff --git a/swadm/src/route.rs b/swadm/src/route.rs index 61ec1bcc..efa9b78f 100644 --- a/swadm/src/route.rs +++ b/swadm/src/route.rs @@ -15,6 +15,7 @@ use colored::*; use futures::stream::TryStreamExt; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use tabwriter::TabWriter; +use uuid::Uuid; use dpd_client::Client; use dpd_client::ClientInfo; @@ -142,6 +143,7 @@ fn print_ipv6_route( async fn route_list( client: &Client, + router_id: Uuid, family: Option, ) -> anyhow::Result<()> { let (v4, v6) = match family { @@ -156,7 +158,7 @@ async fn route_list( print_route_header(&mut tw)?; if v4 { let routes: Vec = client - .route_ipv4_list_stream(None) + .router_route_ipv4_list_stream(&router_id, None) .try_collect() .await .context("failed to get IPv4 routes")?; @@ -166,7 +168,7 @@ async fn route_list( } if v6 { let routes: Vec = client - .route_ipv6_list_stream(None) + .router_route_ipv6_list_stream(&router_id, None) .try_collect() .await .context("failed to get IPv6 routes")?; @@ -177,13 +179,17 @@ async fn route_list( tw.flush().map_err(|e| e.into()) } -async fn route_get(client: &Client, cidr: IpNet) -> anyhow::Result<()> { +async fn route_get( + client: &Client, + router_id: Uuid, + cidr: IpNet, +) -> anyhow::Result<()> { let mut tw = TabWriter::new(stdout()); print_route_header(&mut tw)?; match &cidr { IpNet::V4(c) => { let targets = client - .route_ipv4_get(c) + .router_route_ipv4_get(&router_id, c) .await .context("failed to get IPv4 route")? .into_inner(); @@ -191,7 +197,7 @@ async fn route_get(client: &Client, cidr: IpNet) -> anyhow::Result<()> { } IpNet::V6(c) => { let targets = client - .route_ipv6_get(c) + .router_route_ipv6_get(&router_id, c) .await .context("failed to get IPv6 route")? .into_inner(); @@ -203,6 +209,7 @@ async fn route_get(client: &Client, cidr: IpNet) -> anyhow::Result<()> { async fn route_add( client: &Client, + router_id: Uuid, cidr: IpNet, link_path: LinkPath, gw: IpAddr, @@ -231,27 +238,29 @@ async fn route_add( } }; client - .route_ipv4_add(&types::Ipv4RouteUpdate { - cidr, - target, - replace: false, - }) + .router_route_ipv4_add( + &router_id, + &types::Ipv4RouteUpdate { cidr, target, replace: false }, + ) .await .context("adding IPv4 route") .map(|_| ()) } (IpAddr::V6(tgt_ip), IpNet::V6(cidr)) => client - .route_ipv6_add(&types::Ipv6RouteUpdate { - cidr, - target: types::Ipv6Route { - tag: client.inner().tag.clone(), - port_id: link_path.port_id, - link_id: link_path.link_id, - tgt_ip, - vlan_id, + .router_route_ipv6_add( + &router_id, + &types::Ipv6RouteUpdate { + cidr, + target: types::Ipv6Route { + tag: client.inner().tag.clone(), + port_id: link_path.port_id, + link_id: link_path.link_id, + tgt_ip, + vlan_id, + }, + replace: false, }, - replace: false, - }) + ) .await .context("adding IPv6 route") .map(|_| ()), @@ -263,6 +272,7 @@ async fn route_add( async fn route_del( client: &Client, + router_id: Uuid, cidr: IpNet, link_path: Option, gw: Option, @@ -279,13 +289,15 @@ async fn route_del( let port = link_path.port_id; let link = link_path.link_id; client - .route_ipv4_delete_target(&c, &port, &link, &gw) + .router_route_ipv4_delete_target( + &router_id, &c, &port, &link, &gw, + ) .await .context("deleting IPv4 route target") .map(|_| ()) } None => client - .route_ipv4_delete(&c) + .router_route_ipv4_delete(&router_id, &c) .await .context("deleting IPv4 route") .map(|_| ()), @@ -293,7 +305,7 @@ async fn route_del( IpNet::V6(c) => match gw { Some(_) => Err(anyhow!("not supported")), None => client - .route_ipv6_delete(&c) + .router_route_ipv6_delete(&router_id, &c) .await .context("deleting IPv6 route") .map(|_| ()), @@ -301,15 +313,19 @@ async fn route_del( } } -pub async fn route_cmd(client: &Client, cmd: Route) -> anyhow::Result<()> { +pub async fn route_cmd( + client: &Client, + router_id: Uuid, + cmd: Route, +) -> anyhow::Result<()> { match cmd { - Route::List { family } => route_list(client, family).await, - Route::Get { cidr } => route_get(client, cidr).await, + Route::List { family } => route_list(client, router_id, family).await, + Route::Get { cidr } => route_get(client, router_id, cidr).await, Route::Add { cidr, link_path, gw, vlan_id } => { - route_add(client, cidr, link_path, gw, vlan_id).await + route_add(client, router_id, cidr, link_path, gw, vlan_id).await } Route::Del { cidr, link_path, gw } => { - route_del(client, cidr, link_path, gw).await + route_del(client, router_id, cidr, link_path, gw).await } } } diff --git a/swadm/src/router.rs b/swadm/src/router.rs new file mode 100644 index 00000000..3bc179dd --- /dev/null +++ b/swadm/src/router.rs @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use std::collections::BTreeMap; +use std::net::Ipv6Addr; + +use anyhow::Context; +use clap::Subcommand; +use uuid::Uuid; + +use dpd_client::Client; + +#[derive(Debug, Subcommand)] +/// Manage routers +pub enum Router { + /// List all routers and their endpoints + #[clap(visible_alias = "ls")] + List, + /// Show a router's endpoint + Get { + /// Router uuid + router_id: Uuid, + }, + /// Create a router + Create { + /// Router uuid + router_id: Uuid, + /// Endpoint address + #[clap(long)] + endpoint: Ipv6Addr, + }, + /// Delete a router along with its endpoint and routes + Delete { + /// Router uuid + router_id: Uuid, + }, +} + +pub async fn router_cmd(client: &Client, cmd: Router) -> anyhow::Result<()> { + match cmd { + Router::List => { + let routers = client + .router_list() + .await + .context("failed to list routers")? + .into_inner(); + let routers: BTreeMap = + routers.into_iter().collect(); + for (router_id, endpoint) in routers { + println!("{router_id} {endpoint}"); + } + Ok(()) + } + Router::Get { router_id } => { + let endpoint = client + .router_get(&router_id) + .await + .context("failed to get router")? + .into_inner(); + println!("{endpoint}"); + Ok(()) + } + Router::Create { router_id, endpoint } => client + .router_create(&router_id, &endpoint) + .await + .context("failed to create router") + .map(|_| ()), + Router::Delete { router_id } => client + .router_delete(&router_id) + .await + .context("failed to delete router") + .map(|_| ()), + } +} diff --git a/swadm/tests/cli/expect/route_get_default.txt b/swadm/tests/cli/expect/route_get_default.txt new file mode 100644 index 00000000..b6acbc2d --- /dev/null +++ b/swadm/tests/cli/expect/route_get_default.txt @@ -0,0 +1,2 @@ +Subnet Port Link Gateway Vlan +198.51.100.0/24 rear0 0 192.0.2.1 diff --git a/swadm/tests/cli/expect/router_get.txt b/swadm/tests/cli/expect/router_get.txt new file mode 100644 index 00000000..17b86de9 --- /dev/null +++ b/swadm/tests/cli/expect/router_get.txt @@ -0,0 +1 @@ +fd00:1122::1 diff --git a/swadm/tests/cli/expect/router_get_404.txt b/swadm/tests/cli/expect/router_get_404.txt new file mode 100644 index 00000000..6a56970d --- /dev/null +++ b/swadm/tests/cli/expect/router_get_404.txt @@ -0,0 +1,4 @@ +Error: failed to get router + +Caused by: + Error Response: status: 404 Not Found; \ No newline at end of file diff --git a/swadm/tests/cli/expect/router_list.txt b/swadm/tests/cli/expect/router_list.txt new file mode 100644 index 00000000..b8c3081e --- /dev/null +++ b/swadm/tests/cli/expect/router_list.txt @@ -0,0 +1 @@ +11111111-2222-4333-8444-555555555555 fd00:1122::1 diff --git a/swadm/tests/cli/expect/router_route_list.txt b/swadm/tests/cli/expect/router_route_list.txt new file mode 100644 index 00000000..b6acbc2d --- /dev/null +++ b/swadm/tests/cli/expect/router_route_list.txt @@ -0,0 +1,2 @@ +Subnet Port Link Gateway Vlan +198.51.100.0/24 rear0 0 192.0.2.1 diff --git a/swadm/tests/cli/main.rs b/swadm/tests/cli/main.rs index 722a07a6..d11ea2dc 100644 --- a/swadm/tests/cli/main.rs +++ b/swadm/tests/cli/main.rs @@ -7,4 +7,5 @@ mod cmd; mod counters; mod link_apply; +mod router; mod tx_eq; diff --git a/swadm/tests/cli/router.rs b/swadm/tests/cli/router.rs new file mode 100644 index 00000000..add4fcaf --- /dev/null +++ b/swadm/tests/cli/router.rs @@ -0,0 +1,79 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use anyhow::bail; +use serial_test::serial; + +use crate::cmd; + +const ROUTER: &str = "11111111-2222-4333-8444-555555555555"; +const DEFAULT_ROUTER: &str = "00000000-0000-0000-0000-000000000000"; +const ENDPOINT: &str = "fd00:1122::1"; +const PORT: &str = "rear0"; +const LINK: &str = "rear0/0"; +const SUBNET: &str = "198.51.100.0/24"; +const GW: &str = "192.0.2.1"; + +/// Creates a router, reads it back, and deletes it. +#[test] +#[serial] +#[ignore] +fn router_create_get_delete() -> anyhow::Result<()> { + cmd::swadm(format!("router create {ROUTER} --endpoint {ENDPOINT}"))?; + + cmd::swadm(format!("router get {ROUTER}"))? + .try_expectorate("router_get.txt")?; + cmd::swadm("router list")? + .retain_lines(&crate::among![ROUTER]?) + .try_expectorate("router_list.txt")?; + + cmd::swadm(format!("router delete {ROUTER}"))?; + + let mut out: cmd::Output = cmd::swadm(format!("router get {ROUTER}")) + .expect_err("Get cmd on deleted router should fail") + .try_into()?; + // Remove headers with UUID and timestamp + out.trunc_at("headers:").try_expectorate("router_get_404.txt") +} + +/// Adds a route to a router, and checks that it's listed on that router +/// and not on the default routing table. +#[test] +#[serial] +#[ignore] +fn router_route_add() -> anyhow::Result<()> { + crate::link_apply::delete_link(LINK)?; + + cmd::swadm(format!("link create {PORT} -s 100g --fec rs"))?; + cmd::swadm(format!("router create {ROUTER} --endpoint {ENDPOINT}"))?; + cmd::swadm(format!("route add --router-id {ROUTER} {SUBNET} {LINK} {GW}"))?; + + cmd::swadm(format!("route list v4 --router-id {ROUTER}"))? + .try_expectorate("router_route_list.txt")?; + + let default = cmd::swadm("route list v4")?; + if default.as_ref().contains(SUBNET) { + bail!("Route on a router should not be on the default routing table"); + } + Ok(()) +} + +/// A route command without `--router-id` acts on the default routing table. +#[test] +#[serial] +#[ignore] +fn route_default_routing_table() -> anyhow::Result<()> { + crate::link_apply::delete_link(LINK)?; + + cmd::swadm(format!("link create {PORT} -s 100g --fec rs"))?; + cmd::swadm(format!("route add {SUBNET} {LINK} {GW}"))?; + + cmd::swadm(format!("route get --router-id {DEFAULT_ROUTER} {SUBNET}"))? + .try_expectorate("route_get_default.txt")?; + + cmd::swadm(format!("route del {SUBNET}"))?; + Ok(()) +}