feat: replace SourceFilter with IpNet, add policy routing, remove MASQUERADE

Makefile

@@ -236,9 +236,9 @@ update-startbox: core/target/$(RUST_ARCH)-unknown-linux-musl/$(PROFILE)/startbox
 update-deb: results/$(BASENAME).deb # better than update, but only available from debian
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
 	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
-	$(call mkdir,/media/startos/next/tmp/startos-deb)
+	$(call mkdir,/media/startos/next/var/tmp/startos-deb)
-	$(call cp,results/$(BASENAME).deb,/media/startos/next/tmp/startos-deb/$(BASENAME).deb)
+	$(call cp,results/$(BASENAME).deb,/media/startos/next/var/tmp/startos-deb/$(BASENAME).deb)
-	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y --reinstall /tmp/startos-deb/$(BASENAME).deb"')
+	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y --reinstall /var/tmp/startos-deb/$(BASENAME).deb"')
 
 update-squashfs: results/$(BASENAME).squashfs
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi

build/lib/scripts/forward-port

@@ -5,7 +5,7 @@ if [ -z "$sip" ] || [ -z "$dip" ] || [ -z "$dprefix" ] || [ -z "$sport" ] || [ -
     exit 1
 fi
 
-NAME="F$(echo "$sip:$sport -> $dip/$dprefix:$dport ${src_subnet:-any} ${excluded_src:-none}" | sha256sum | head -c 15)"
+NAME="F$(echo "$sip:$sport -> $dip/$dprefix:$dport ${src_subnet:-any}" | sha256sum | head -c 15)"
 
 for kind in INPUT FORWARD ACCEPT; do
     if ! iptables -C $kind -j "${NAME}_${kind}" 2> /dev/null; then
@@ -13,7 +13,7 @@ for kind in INPUT FORWARD ACCEPT; do
         iptables -A $kind -j "${NAME}_${kind}"
     fi
 done
-for kind in PREROUTING INPUT OUTPUT POSTROUTING; do
+for kind in PREROUTING OUTPUT; do
     if ! iptables -t nat -C $kind -j "${NAME}_${kind}" 2> /dev/null; then
         iptables -t nat -N "${NAME}_${kind}" 2> /dev/null
         iptables -t nat -A $kind -j "${NAME}_${kind}"
@@ -26,7 +26,7 @@ trap 'err=1' ERR
 for kind in INPUT FORWARD ACCEPT; do
     iptables -F "${NAME}_${kind}" 2> /dev/null
 done
-for kind in PREROUTING INPUT OUTPUT POSTROUTING; do
+for kind in PREROUTING OUTPUT; do
     iptables -t nat -F "${NAME}_${kind}" 2> /dev/null
 done
 if [ "$UNDO" = 1 ]; then
@@ -37,15 +37,7 @@ fi
 
 # DNAT: rewrite destination for incoming packets (external traffic)
 # When src_subnet is set, only forward traffic from that subnet (private forwards)
-# excluded_src: comma-separated gateway/router IPs to reject (they may masquerade internet traffic)
 if [ -n "$src_subnet" ]; then
-    if [ -n "$excluded_src" ]; then
-        IFS=',' read -ra EXCLUDED <<< "$excluded_src"
-        for excl in "${EXCLUDED[@]}"; do
-            iptables -t nat -A ${NAME}_PREROUTING -s "$excl" -d "$sip" -p tcp --dport "$sport" -j RETURN
-            iptables -t nat -A ${NAME}_PREROUTING -s "$excl" -d "$sip" -p udp --dport "$sport" -j RETURN
-        done
-    fi
     iptables -t nat -A ${NAME}_PREROUTING -s "$src_subnet" -d "$sip" -p tcp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
     iptables -t nat -A ${NAME}_PREROUTING -s "$src_subnet" -d "$sip" -p udp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
 else
@@ -57,11 +49,6 @@ fi
 iptables -t nat -A ${NAME}_OUTPUT -d "$sip" -p tcp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
 iptables -t nat -A ${NAME}_OUTPUT -d "$sip" -p udp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
 
-# MASQUERADE: rewrite source for all forwarded traffic to the destination
-# This ensures responses are routed back through the host regardless of source IP
-iptables -t nat -A ${NAME}_POSTROUTING -d "$dip" -p tcp --dport "$dport" -j MASQUERADE
-iptables -t nat -A ${NAME}_POSTROUTING -d "$dip" -p udp --dport "$dport" -j MASQUERADE
-
 # Allow new connections to be forwarded to the destination
 iptables -A ${NAME}_FORWARD -d $dip -p tcp --dport $dport -m state --state NEW -j ACCEPT
 iptables -A ${NAME}_FORWARD -d $dip -p udp --dport $dport -m state --state NEW -j ACCEPT

core/src/net/forward.rs

@@ -3,6 +3,8 @@ use std::net::{IpAddr, SocketAddrV4};
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
+use ipnet::IpNet;
+
 use futures::channel::oneshot;
 use iddqd::{IdOrdItem, IdOrdMap};
 use rand::Rng;
@@ -47,16 +49,6 @@ impl std::fmt::Display for ForwardRequirements {
     }
 }
 
-/// Source-IP filter for private forwards: restricts traffic to a subnet
-/// while excluding gateway/router IPs that may masquerade internet traffic.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct SourceFilter {
-    /// Network CIDR to allow (e.g. "192.168.1.0/24")
-    subnet: String,
-    /// Comma-separated gateway IPs to exclude (they may masquerade internet traffic)
-    excluded: String,
-}
-
 #[derive(Debug, Deserialize, Serialize)]
 pub struct AvailablePorts(BTreeMap<u16, bool>);
 impl AvailablePorts {
@@ -129,7 +121,7 @@ struct ForwardMapping {
     source: SocketAddrV4,
     target: SocketAddrV4,
     target_prefix: u8,
-    src_filter: Option<SourceFilter>,
+    src_filter: Option<IpNet>,
     rc: Weak<()>,
 }
 
@@ -144,7 +136,7 @@ impl PortForwardState {
         source: SocketAddrV4,
         target: SocketAddrV4,
         target_prefix: u8,
-        src_filter: Option<SourceFilter>,
+        src_filter: Option<IpNet>,
     ) -> Result<Arc<()>, Error> {
         if let Some(existing) = self.mappings.get_mut(&source) {
             if existing.target == target && existing.src_filter == src_filter {
@@ -241,7 +233,7 @@ enum PortForwardCommand {
         source: SocketAddrV4,
         target: SocketAddrV4,
         target_prefix: u8,
-        src_filter: Option<SourceFilter>,
+        src_filter: Option<IpNet>,
         respond: oneshot::Sender<Result<Arc<()>, Error>>,
     },
     Gc {
@@ -358,7 +350,7 @@ impl PortForwardController {
         source: SocketAddrV4,
         target: SocketAddrV4,
         target_prefix: u8,
-        src_filter: Option<SourceFilter>,
+        src_filter: Option<IpNet>,
     ) -> Result<Arc<()>, Error> {
         let (send, recv) = oneshot::channel();
         self.req
@@ -455,19 +447,7 @@ impl InterfaceForwardEntry {
                                 if reqs.public_gateways.contains(gw_id) {
                                     None
                                 } else if reqs.private_ips.contains(&IpAddr::V4(ip)) {
-                                    let excluded = ip_info
+                                    Some(subnet.trunc())
-                                        .lan_ip
-                                        .iter()
-                                        .filter_map(|ip| match ip {
-                                            IpAddr::V4(v4) => Some(v4.to_string()),
-                                            _ => None,
-                                        })
-                                        .collect::<Vec<_>>()
-                                        .join(",");
-                                    Some(SourceFilter {
-                                        subnet: subnet.trunc().to_string(),
-                                        excluded,
-                                    })
                                 } else {
                                     continue;
                                 };
@@ -725,7 +705,7 @@ async fn forward(
     source: SocketAddrV4,
     target: SocketAddrV4,
     target_prefix: u8,
-    src_filter: Option<&SourceFilter>,
+    src_filter: Option<&IpNet>,
 ) -> Result<(), Error> {
     let mut cmd = Command::new("/usr/lib/startos/scripts/forward-port");
     cmd.env("sip", source.ip().to_string())
@@ -733,11 +713,8 @@ async fn forward(
         .env("dprefix", target_prefix.to_string())
         .env("sport", source.port().to_string())
         .env("dport", target.port().to_string());
-    if let Some(filter) = src_filter {
+    if let Some(subnet) = src_filter {
-        cmd.env("src_subnet", &filter.subnet);
+        cmd.env("src_subnet", subnet.to_string());
-        if !filter.excluded.is_empty() {
-            cmd.env("excluded_src", &filter.excluded);
-        }
     }
     cmd.invoke(ErrorKind::Network).await?;
     Ok(())
@@ -747,7 +724,7 @@ async fn unforward(
     source: SocketAddrV4,
     target: SocketAddrV4,
     target_prefix: u8,
-    src_filter: Option<&SourceFilter>,
+    src_filter: Option<&IpNet>,
 ) -> Result<(), Error> {
     let mut cmd = Command::new("/usr/lib/startos/scripts/forward-port");
     cmd.env("UNDO", "1")
@@ -756,11 +733,8 @@ async fn unforward(
         .env("dprefix", target_prefix.to_string())
         .env("sport", source.port().to_string())
         .env("dport", target.port().to_string());
-    if let Some(filter) = src_filter {
+    if let Some(subnet) = src_filter {
-        cmd.env("src_subnet", &filter.subnet);
+        cmd.env("src_subnet", subnet.to_string());
-        if !filter.excluded.is_empty() {
-            cmd.env("excluded_src", &filter.excluded);
-        }
     }
     cmd.invoke(ErrorKind::Network).await?;
     Ok(())

core/src/net/gateway.rs

@@ -657,6 +657,62 @@ async fn watch_ip(
                                 None
                             };
 
+                            // Policy routing: track per-interface table for cleanup on scope exit
+                            let policy_table_id = if !matches!(
+                                device_type,
+                                Some(
+                                    NetworkInterfaceType::Bridge
+                                        | NetworkInterfaceType::Loopback
+                                )
+                            ) {
+                                if_nametoindex(iface.as_str())
+                                    .map(|idx| 1000 + idx)
+                                    .log_err()
+                            } else {
+                                None
+                            };
+                            struct PolicyRoutingCleanup {
+                                table_id: u32,
+                                iface: String,
+                            }
+                            impl Drop for PolicyRoutingCleanup {
+                                fn drop(&mut self) {
+                                    let table_str = self.table_id.to_string();
+                                    let iface = std::mem::take(&mut self.iface);
+                                    tokio::spawn(async move {
+                                        Command::new("ip")
+                                            .arg("rule").arg("del")
+                                            .arg("fwmark").arg(&table_str)
+                                            .arg("lookup").arg(&table_str)
+                                            .invoke(ErrorKind::Network)
+                                            .await
+                                            .log_err();
+                                        Command::new("ip")
+                                            .arg("route").arg("flush")
+                                            .arg("table").arg(&table_str)
+                                            .invoke(ErrorKind::Network)
+                                            .await
+                                            .log_err();
+                                        Command::new("iptables")
+                                            .arg("-t").arg("mangle")
+                                            .arg("-D").arg("PREROUTING")
+                                            .arg("-i").arg(&iface)
+                                            .arg("-m").arg("conntrack")
+                                            .arg("--ctstate").arg("NEW")
+                                            .arg("-j").arg("CONNMARK")
+                                            .arg("--set-mark").arg(&table_str)
+                                            .invoke(ErrorKind::Network)
+                                            .await
+                                            .log_err();
+                                    });
+                                }
+                            }
+                            let _policy_guard: Option<PolicyRoutingCleanup> =
+                                policy_table_id.map(|t| PolicyRoutingCleanup {
+                                    table_id: t,
+                                    iface: iface.as_str().to_owned(),
+                                });
+
                             loop {
                                 until
                                     .run(async {
@@ -666,7 +722,7 @@ async fn watch_ip(
                                             .into_iter()
                                             .chain(ip6_proxy.address_data().await?)
                                             .collect_vec();
-                                        let lan_ip = [
+                                        let lan_ip: OrdSet<IpAddr> = [
                                             Some(ip4_proxy.gateway().await?)
                                                 .filter(|g| !g.is_empty())
                                                 .and_then(|g| g.parse::<IpAddr>().log_err()),
@@ -703,22 +759,122 @@ async fn watch_ip(
                                             .into_iter()
                                             .map(IpNet::try_from)
                                             .try_collect()?;
-                                        // let tables = ip4_proxy.route_data().await?.into_iter().filter_map(|d|d.table).collect::<Vec<_>>();
+                                        // Policy routing: ensure replies exit the same interface
-                                        // if !tables.is_empty() {
+                                        // they arrived on, eliminating the need for MASQUERADE.
-                                        //     let rules = String::from_utf8(Command::new("ip").arg("rule").arg("list").invoke(ErrorKind::Network).await?)?;
+                                        if let Some(guard) = &_policy_guard {
-                                        //     for table in tables {
+                                            let table_id = guard.table_id;
-                                        //         for subnet in subnets.iter().filter(|s| s.addr().is_ipv4()) {
+                                            let table_str = table_id.to_string();
-                                        //             let subnet_string = subnet.trunc().to_string();
+
-                                        //             let rule = ["from", &subnet_string, "lookup", &table.to_string()];
+                                            let ipv4_gateway: Option<Ipv4Addr> =
-                                        //             if !rules.contains(&rule.join(" ")) {
+                                                lan_ip.iter().find_map(|ip| match ip {
-                                        //                 if rules.contains(&rule[..2].join(" ")) {
+                                                    IpAddr::V4(v4) => Some(v4),
-                                        //                     Command::new("ip").arg("rule").arg("del").args(&rule[..2]).invoke(ErrorKind::Network).await?;
+                                                    _ => None,
-                                        //                 }
+                                                }).copied();
-                                        //                 Command::new("ip").arg("rule").arg("add").args(rule).invoke(ErrorKind::Network).await?;
+                                            let ipv4_subnets: Vec<IpNet> = subnets
-                                        //             }
+                                                .iter()
-                                        //         }
+                                                .filter(|s| s.addr().is_ipv4())
-                                        //     }
+                                                .cloned()
-                                        // }
+                                                .collect();
+
+                                            // Flush and rebuild per-interface routing table
+                                            Command::new("ip")
+                                                .arg("route").arg("flush")
+                                                .arg("table").arg(&table_str)
+                                                .invoke(ErrorKind::Network)
+                                                .await
+                                                .log_err();
+                                            for subnet in &ipv4_subnets {
+                                                let subnet_str = subnet.trunc().to_string();
+                                                Command::new("ip")
+                                                    .arg("route").arg("add").arg(&subnet_str)
+                                                    .arg("dev").arg(iface.as_str())
+                                                    .arg("table").arg(&table_str)
+                                                    .invoke(ErrorKind::Network)
+                                                    .await
+                                                    .log_err();
+                                            }
+                                            {
+                                                let mut cmd = Command::new("ip");
+                                                cmd.arg("route").arg("add").arg("default");
+                                                if let Some(gw) = ipv4_gateway {
+                                                    cmd.arg("via").arg(gw.to_string());
+                                                }
+                                                cmd.arg("dev").arg(iface.as_str())
+                                                    .arg("table").arg(&table_str);
+                                                if ipv4_gateway.is_none() {
+                                                    cmd.arg("scope").arg("link");
+                                                }
+                                                cmd.invoke(ErrorKind::Network)
+                                                    .await
+                                                    .log_err();
+                                            }
+
+                                            // Ensure global CONNMARK restore rule in mangle PREROUTING
+                                            // (restores fwmark from conntrack mark on reply packets)
+                                            if !Command::new("iptables")
+                                                .arg("-t").arg("mangle")
+                                                .arg("-C").arg("PREROUTING")
+                                                .arg("-j").arg("CONNMARK")
+                                                .arg("--restore-mark")
+                                                .status().await
+                                                .map_or(false, |s| s.success())
+                                            {
+                                                Command::new("iptables")
+                                                    .arg("-t").arg("mangle")
+                                                    .arg("-I").arg("PREROUTING").arg("1")
+                                                    .arg("-j").arg("CONNMARK")
+                                                    .arg("--restore-mark")
+                                                    .invoke(ErrorKind::Network)
+                                                    .await
+                                                    .log_err();
+                                            }
+
+                                            // Mark NEW connections arriving on this interface
+                                            // with its routing table ID via conntrack mark
+                                            if !Command::new("iptables")
+                                                .arg("-t").arg("mangle")
+                                                .arg("-C").arg("PREROUTING")
+                                                .arg("-i").arg(iface.as_str())
+                                                .arg("-m").arg("conntrack")
+                                                .arg("--ctstate").arg("NEW")
+                                                .arg("-j").arg("CONNMARK")
+                                                .arg("--set-mark").arg(&table_str)
+                                                .status().await
+                                                .map_or(false, |s| s.success())
+                                            {
+                                                Command::new("iptables")
+                                                    .arg("-t").arg("mangle")
+                                                    .arg("-A").arg("PREROUTING")
+                                                    .arg("-i").arg(iface.as_str())
+                                                    .arg("-m").arg("conntrack")
+                                                    .arg("--ctstate").arg("NEW")
+                                                    .arg("-j").arg("CONNMARK")
+                                                    .arg("--set-mark").arg(&table_str)
+                                                    .invoke(ErrorKind::Network)
+                                                    .await
+                                                    .log_err();
+                                            }
+
+                                            // Ensure fwmark-based ip rule for this interface's table
+                                            let rules_output = String::from_utf8(
+                                                Command::new("ip")
+                                                    .arg("rule").arg("list")
+                                                    .invoke(ErrorKind::Network)
+                                                    .await?,
+                                            )?;
+                                            if !rules_output.lines().any(|l| {
+                                                l.contains("fwmark")
+                                                    && l.contains(&format!("lookup {table_id}"))
+                                            }) {
+                                                Command::new("ip")
+                                                    .arg("rule").arg("add")
+                                                    .arg("fwmark").arg(&table_str)
+                                                    .arg("lookup").arg(&table_str)
+                                                    .invoke(ErrorKind::Network)
+                                                    .await
+                                                    .log_err();
+                                            }
+                                        }
                                         let wan_ip = if !subnets.is_empty()
                                             && !matches!(
                                                 device_type,

debian/dpkg-build.sh

@@ -23,7 +23,7 @@ if [ "${PROJECT}" = "startos" ]; then
 else
     INSTALL_TARGET="install-${PROJECT#start-}"
 fi
-make "${INSTALL_TARGET}" DESTDIR=dpkg-workdir/$BASENAME
+make "${INSTALL_TARGET}" DESTDIR=dpkg-workdir/$BASENAME REMOTE=
 
 if [ -f dpkg-workdir/$BASENAME/usr/lib/$PROJECT/depends ]; then
     if [ -n "$DEPENDS" ]; then