Encrypt VLAN/VXLAN tunnel traffic with per-tunnel AES-256 keys

members/nullnet-client/Cargo.toml

@@ -23,4 +23,5 @@ futures = "0.3.32"
 network-interface = "2.0.5"
 gag.workspace = true
 chrono.workspace = true
-nfq = "0.2"
+nfq = "0.2"
+aes-gcm = "0.10"

members/nullnet-client/src/commands/mod.rs

@@ -25,11 +25,20 @@ pub(crate) async fn setup_br0(rtnetlink_handle: &RtNetLinkHandle) {
     // delete existing OpenFlow rules
     OvsCommand::DeleteFlows.execute();
 
-    // use the built-in switching logic
-    OvsCommand::AddFlow.execute();
-
-    // add our TAP to the bridge as a trunk port
+    // add our TAP to the bridge as a trunk port first, so the flow rules
+    // below can reference it by name
     OvsCommand::AddTrunkPort.execute();
+
+    // Safe fallback (same as OVS's original single default rule) for
+    // anything not covered by a more specific rule — mainly the brief
+    // window between an access port being created and its own redirect
+    // rule (installed in `configure_access_port`) landing.
+    OvsCommand::AddDefaultFlow.execute();
+
+    // Traffic arriving from the trunk (i.e. already decrypted by
+    // nullnet-client's userspace forwarder) is delivered by normal
+    // VLAN-aware switching.
+    OvsCommand::AddTrunkDeliveryFlow.execute();
 }
 
 pub(crate) async fn configure_access_port(
@@ -51,9 +60,22 @@ pub(crate) async fn configure_access_port(
 
     // add the peer interface to the bridge as an access port
     OvsCommand::AddAccessPort(&veth_peer_name, vlan_id).execute();
+
+    // Redirect this port's traffic to the trunk instead of letting OVS
+    // switch it directly to another local access port — and re-add the
+    // 802.1Q tag that gets stripped along the way, since the raw `output`
+    // action used to reach the trunk doesn't do that automatically the way
+    // `actions=normal` would.
+    OvsCommand::AddAccessRedirectFlow(&veth_peer_name, vlan_id).execute();
 }
 
 pub(crate) async fn remove_vlan(rtnetlink_handle: &RtNetLinkHandle, vlan_id: u16) {
+    // remove this port's redirect flow before the port itself disappears,
+    // so no stale rule is left behind that could later match a different,
+    // unrelated port reusing the same OVS port number
+    let veth_peer_name = format!("veth-{vlan_id}p");
+    OvsCommand::DeleteAccessRedirectFlow(&veth_peer_name).execute();
+
     // delete the veth pair
     rtnetlink_handle
         .execute(NetLinkCommand::DeleteVeth(vlan_id))

members/nullnet-client/src/commands/ovs.rs

@@ -7,7 +7,30 @@ pub(super) enum OvsCommand<'a> {
     DeleteBridge,
     AddBridge,
     DeleteFlows,
-    AddFlow,
+    /// Fallback for anything not covered by a more specific rule below —
+    /// same behavior as OVS's original single default flow. Mainly covers
+    /// the brief startup window between an access port being created and
+    /// its own redirect flow (below) landing.
+    AddDefaultFlow,
+    /// Traffic arriving from the trunk (already decrypted by nullnet-client's
+    /// userspace forwarder) gets delivered by normal VLAN-aware L2 switching.
+    AddTrunkDeliveryFlow,
+    /// One rule per access port, installed alongside it: redirect this
+    /// port's traffic to the trunk instead of letting OVS switch it
+    /// directly to another local access port (which would bypass the TAP
+    /// and the encrypting userspace forwarder entirely when a tunnel's two
+    /// endpoints happen to be colocated on this host). `output:<port>` is a
+    /// raw action — unlike `actions=normal`, it does *not* re-add the
+    /// 802.1Q tag that access ports carry only internally, so this
+    /// explicitly pushes the tag back on first: without that, packets would
+    /// arrive at nullnet-client's TAP already stripped of their VLAN tag
+    /// and get silently dropped as malformed.
+    AddAccessRedirectFlow(&'a str, u16),
+    /// Removes exactly the rule `AddAccessRedirectFlow` installed for this
+    /// port, so a torn-down tunnel doesn't leave a stale flow entry that
+    /// could wrongly match a future, unrelated port reusing the same
+    /// OVS port number.
+    DeleteAccessRedirectFlow(&'a str),
     AddTrunkPort,
     AddAccessPort(&'a str, u16),
 }
@@ -33,7 +56,11 @@ impl OvsCommand<'_> {
             | OvsCommand::DeleteBridge
             | OvsCommand::AddAccessPort(_, _)
             | OvsCommand::AddTrunkPort => "ovs-vsctl",
-            OvsCommand::DeleteFlows | OvsCommand::AddFlow => "ovs-ofctl",
+            OvsCommand::DeleteFlows
+            | OvsCommand::AddDefaultFlow
+            | OvsCommand::AddTrunkDeliveryFlow
+            | OvsCommand::AddAccessRedirectFlow(_, _)
+            | OvsCommand::DeleteAccessRedirectFlow(_) => "ovs-ofctl",
         }
     }
 
@@ -45,10 +72,34 @@ impl OvsCommand<'_> {
                 .iter()
                 .map(ToString::to_string)
                 .collect(),
-            OvsCommand::AddFlow => ["add-flow", "br0", "priority=0,actions=normal"]
+            OvsCommand::AddDefaultFlow => ["add-flow", "br0", "priority=0,actions=normal"]
                 .iter()
                 .map(ToString::to_string)
                 .collect(),
+            OvsCommand::AddTrunkDeliveryFlow => [
+                "add-flow",
+                "br0",
+                &format!("priority=200,in_port={TAP_NAME},actions=normal"),
+            ]
+            .iter()
+            .map(ToString::to_string)
+            .collect(),
+            OvsCommand::AddAccessRedirectFlow(dev, vlan) => [
+                "add-flow",
+                "br0",
+                &format!(
+                    "priority=150,in_port={dev},actions=push_vlan:0x8100,mod_vlan_vid:{vlan},output:{TAP_NAME}"
+                ),
+            ]
+            .iter()
+            .map(ToString::to_string)
+            .collect(),
+            OvsCommand::DeleteAccessRedirectFlow(dev) => {
+                ["del-flows", "br0", &format!("in_port={dev}")]
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect()
+            }
             OvsCommand::AddTrunkPort => ["add-port", "br0", TAP_NAME]
                 .iter()
                 .map(ToString::to_string)

members/nullnet-client/src/control_channel.rs

@@ -162,6 +162,24 @@ async fn handle_vlan_setup(
                 }),
             );
         })?;
+    // Fail before touching the network if the key is malformed — running
+    // this tunnel without a valid key would mean forwarding traffic in the
+    // clear instead of encrypted.
+    let encryption_key: [u8; 32] = message
+        .encryption_key
+        .try_into()
+        .map_err(|_| "VLAN setup message carried a malformed encryption key")
+        .handle_err(location!())
+        .inspect_err(|e| {
+            fire_event(
+                &grpc,
+                AgentEventKind::VlanSetupFailed(AgentVlanSetupFailed {
+                    vlan_id: message.vlan_id,
+                    local_veth: local_veth.to_string(),
+                    error_reason: e.to_str().to_string(),
+                }),
+            );
+        })?;
 
     // setup VLAN on this machine
     let init_t = std::time::Instant::now();
@@ -176,11 +194,12 @@ async fn handle_vlan_setup(
         init_t.elapsed().as_millis()
     );
 
-    // register peer
-    peers
-        .write()
-        .await
-        .insert(VethKey::new(remote_veth, vlan_id), remote_ip);
+    // register peer + this tunnel's encryption key
+    {
+        let mut peers = peers.write().await;
+        peers.insert(VethKey::new(remote_veth, vlan_id), remote_ip);
+        peers.insert_key(vlan_id, &encryption_key);
+    }
 
     // add host mapping if needed
     if let Some(host_mapping) = &message.host_mapping {
@@ -288,6 +307,25 @@ async fn handle_vxlan_setup(
         .remote_ip
         .parse::<Ipv4Addr>()
         .handle_err(location!())?;
+    // Fail before touching the network if the key is malformed — running
+    // this tunnel without a valid key would mean forwarding traffic in the
+    // clear instead of encrypted.
+    let encryption_key: [u8; 32] = message
+        .encryption_key
+        .try_into()
+        .map_err(|_| "VXLAN setup message carried a malformed encryption key")
+        .handle_err(location!())
+        .inspect_err(|e| {
+            fire_event(
+                &grpc,
+                AgentEventKind::VxlanSetupFailed(AgentVxlanSetupFailed {
+                    vxlan_id,
+                    ns_name: ns_name.clone(),
+                    error_code: -1,
+                }),
+            );
+            eprintln!("[vxlan_setup] {}", e.to_str());
+        })?;
 
     // setup VXLAN on this machine (optionally attaching a Docker container)
     let init_t = std::time::Instant::now();
@@ -298,7 +336,9 @@ async fn handle_vxlan_setup(
         .arg(br_name)
         .arg(br_net.to_string())
         .arg(local_ip.to_string())
-        .arg(remote_ip.to_string());
+        .arg(remote_ip.to_string())
+        .arg(hex_encode(&encryption_key))
+        .arg(message.dstport.to_string());
     if let Some(container) = &message.docker_container {
         cmd.arg(container);
     }
@@ -459,7 +499,12 @@ fn handle_vxlan_teardown(
     let br_name = message.br_name;
 
     let mut cmd = std::process::Command::new("./vxlan_scripts/vxlan-teardown.sh");
-    cmd.arg(vxlan_id.to_string()).arg(&ns_name).arg(&br_name);
+    cmd.arg(vxlan_id.to_string())
+        .arg(&ns_name)
+        .arg(&br_name)
+        .arg(&message.local_ip)
+        .arg(&message.remote_ip)
+        .arg(message.dstport.to_string());
     if let Some(container) = &message.docker_container {
         cmd.arg(container);
     }
@@ -710,3 +755,9 @@ fn remove_hosts_entry(content: &str, name: &str) -> String {
         .collect();
     lines.join("\n") + "\n"
 }
+
+/// Lowercase hex encoding, used to pass the tunnel's AES key to
+/// `vxlan-setup.sh`/`vxlan-teardown.sh` as a shell argument.
+fn hex_encode(bytes: &[u8]) -> String {
+    bytes.iter().map(|b| format!("{b:02x}")).collect()
+}

members/nullnet-client/src/craft/reject_payloads.rs

@@ -3,66 +3,45 @@ use etherparse::{
     Icmpv4Header, Icmpv4Type, IpFragOffset, IpNumber, LaxPacketHeaders, LinkExtHeader, LinkHeader,
     NetHeaders, TcpOptions, TransportHeader,
 };
-use std::net::SocketAddr;
-use std::sync::Arc;
-use tokio::net::UdpSocket;
 
-/// Sends a proper message to gracefully acknowledge a peer that a packet was rejected,
+/// Builds a proper message to gracefully acknowledge a peer that a packet was rejected,
 /// based on the observed protocol:
 /// - in case of TCP, a packet with RST and ACK flag is sent
 /// - in case of UDP, an ICMP port unreachable message is sent
 /// - in case of other protocols, an ICMP host unreachable message is sent
-pub async fn send_termination_message(
-    packet: &[u8],
-    socket: &Arc<UdpSocket>,
-    remote_socket: SocketAddr,
-) {
-    let Ok(headers) = LaxPacketHeaders::from_ethernet(packet) else {
-        return;
-    };
+///
+/// Returns the plaintext Ethernet frame to send; the caller (`forward/receive.rs`)
+/// is responsible for encrypting and framing it before it hits the wire, same as
+/// any other outbound frame on the VLAN forwarder.
+pub fn build_termination_message(packet: &[u8]) -> Option<Vec<u8>> {
+    let headers = LaxPacketHeaders::from_ethernet(packet).ok()?;
     let Some(NetHeaders::Ipv4(ip_header, _)) = &headers.net else {
-        return;
+        return None;
     };
     let IpNumber(proto) = ip_header.protocol;
 
     match proto {
-        6 => Box::pin(send_tcp_rst(headers, socket, remote_socket)).await,
+        6 => send_tcp_rst(headers),
         17 => {
             // port unreachable
             let icmp_type = Icmpv4Type::DestinationUnreachable(DestUnreachableHeader::Port);
-            Box::pin(send_destination_unreachable(
-                packet,
-                headers,
-                socket,
-                icmp_type,
-                remote_socket,
-            ))
-            .await;
+            send_destination_unreachable(packet, headers, icmp_type)
         }
         _ => {
             // host unreachable
             let icmp_type = Icmpv4Type::DestinationUnreachable(DestUnreachableHeader::Host);
-            Box::pin(send_destination_unreachable(
-                packet,
-                headers,
-                socket,
-                icmp_type,
-                remote_socket,
-            ))
-            .await;
+            send_destination_unreachable(packet, headers, icmp_type)
         }
     }
 }
 
-async fn send_destination_unreachable(
+fn send_destination_unreachable(
     packet: &[u8],
     headers: LaxPacketHeaders<'_>,
-    socket: &Arc<UdpSocket>,
     icmp_type: Icmpv4Type,
-    remote_socket: SocketAddr,
-) {
+) -> Option<Vec<u8>> {
     let Some(LinkHeader::Ethernet2(mut ethernet_header)) = headers.link else {
-        return;
+        return None;
     };
     std::mem::swap(
         &mut ethernet_header.source,
@@ -80,7 +59,7 @@ async fn send_destination_unreachable(
         .collect();
 
     let Some(NetHeaders::Ipv4(mut ip_header, _)) = headers.net else {
-        return;
+        return None;
     };
     let original_ip_header_bytes = ip_header.to_bytes();
     let size_up_to_ip_header =
@@ -112,19 +91,12 @@ async fn send_destination_unreachable(
         &icmp_payload[..],
     ].concat();
 
-    socket
-        .send_to(&pkt_response, remote_socket)
-        .await
-        .unwrap_or(0);
+    Some(pkt_response)
 }
 
-async fn send_tcp_rst(
-    headers: LaxPacketHeaders<'_>,
-    socket: &Arc<UdpSocket>,
-    remote_socket: SocketAddr,
-) {
+fn send_tcp_rst(headers: LaxPacketHeaders<'_>) -> Option<Vec<u8>> {
     let Some(LinkHeader::Ethernet2(mut ethernet_header)) = headers.link else {
-        return;
+        return None;
     };
     std::mem::swap(
         &mut ethernet_header.source,
@@ -142,7 +114,7 @@ async fn send_tcp_rst(
         .collect();
 
     let Some(NetHeaders::Ipv4(mut ip_header, _)) = headers.net else {
-        return;
+        return None;
     };
     ip_header.identification = 0;
     ip_header.fragment_offset = IpFragOffset::ZERO;
@@ -152,7 +124,7 @@ async fn send_tcp_rst(
     let ip_header_bytes = ip_header.to_bytes();
 
     let Some(TransportHeader::Tcp(mut tcp_header)) = headers.transport else {
-        return;
+        return None;
     };
     let src_port_orig = tcp_header.source_port;
     let seq_num_orig = tcp_header.sequence_number;
@@ -187,8 +159,5 @@ async fn send_tcp_rst(
         &tcp_header_bytes[..],
     ].concat();
 
-    socket
-        .send_to(&pkt_response, remote_socket)
-        .await
-        .unwrap_or(0);
+    Some(pkt_response)
 }