From db4a4a59bdbbbee937876b2363c3b237cb36404d Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 13 Feb 2026 18:20:33 +0100 Subject: [PATCH 1/2] mesh: set preferred source for WireGuard routes Set Route.Src for routes installed via the kilo interface to the node private IP when available. Without an explicit source, the kernel may pick the WireGuard overlay address (for example 100.66.0.x). In environments like Azure SDN this can break return traffic because the overlay address is not routable by the underlay. Update route expectations in TestRoutes to assert Src across the affected logical and full topology cases. Signed-off-by: Andrei Kvapil --- pkg/mesh/routes.go | 12 +++++ pkg/mesh/routes_test.go | 97 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/pkg/mesh/routes.go b/pkg/mesh/routes.go index 1f6b9aa1..0c00be6f 100644 --- a/pkg/mesh/routes.go +++ b/pkg/mesh/routes.go @@ -135,6 +135,14 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface } return routes, rules } + // Compute the preferred source address for routes through the WireGuard interface. + // Without this, the kernel picks the WireGuard overlay IP (e.g. 100.66.0.x) as the + // source, which can cause issues in environments like Azure SDN where the overlay + // IP is unknown to the network fabric and reply packets cannot be routed back. + var src net.IP + if t.privateIP != nil { + src = t.privateIP.IP + } for _, segment := range t.segments { // Add routes for the current segment if local is true. if segment.location == t.location { @@ -190,6 +198,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface Flags: int(netlink.FLAG_ONLINK), Gw: segment.wireGuardIP, LinkIndex: kiloIface, + Src: src, Protocol: unix.RTPROT_STATIC, }) // Don't add routes through Kilo if the private IP @@ -207,6 +216,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface Flags: int(netlink.FLAG_ONLINK), Gw: segment.wireGuardIP, LinkIndex: kiloIface, + Src: src, Protocol: unix.RTPROT_STATIC, }) } @@ -218,6 +228,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface Flags: int(netlink.FLAG_ONLINK), Gw: segment.wireGuardIP, LinkIndex: kiloIface, + Src: src, Protocol: unix.RTPROT_STATIC, }) } @@ -228,6 +239,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface routes = append(routes, &netlink.Route{ Dst: &peer.AllowedIPs[i], LinkIndex: kiloIface, + Src: src, Protocol: unix.RTPROT_STATIC, }) } diff --git a/pkg/mesh/routes_test.go b/pkg/mesh/routes_test.go index b9157752..aca1182a 100644 --- a/pkg/mesh/routes_test.go +++ b/pkg/mesh/routes_test.go @@ -51,6 +51,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -58,6 +59,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -65,6 +67,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -72,6 +75,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -79,6 +83,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -86,21 +91,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -115,6 +124,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -122,6 +132,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -129,21 +140,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -299,6 +314,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -306,6 +322,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -313,6 +330,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -320,6 +338,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -327,6 +346,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -334,21 +354,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[3].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -363,6 +387,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -370,6 +395,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -377,6 +403,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -384,6 +411,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -391,21 +419,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[3].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -420,6 +452,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -427,6 +460,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -434,6 +468,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -441,6 +476,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -448,6 +484,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -455,21 +492,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[3].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -485,6 +526,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -492,6 +534,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -499,6 +542,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -506,6 +550,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -513,6 +558,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -520,21 +566,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -550,6 +600,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -557,6 +608,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -564,6 +616,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -571,6 +624,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -578,6 +632,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -585,21 +640,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -615,6 +674,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -622,6 +682,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -636,21 +697,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -666,6 +731,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -673,6 +739,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -695,21 +762,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(LogicalGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -908,6 +979,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -915,6 +987,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -922,6 +995,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -929,6 +1003,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -936,6 +1011,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -943,21 +1019,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[3].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["a"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -973,6 +1053,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -980,6 +1061,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -987,6 +1069,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -994,6 +1077,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[2].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -1001,21 +1085,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["b"].Name).segments[3].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, @@ -1031,6 +1119,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -1038,6 +1127,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[0].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -1045,6 +1135,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -1052,6 +1143,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -1059,6 +1151,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -1066,21 +1159,25 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[3].wireGuardIP, LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["a"].AllowedIPs[1], LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { Dst: &peers["b"].AllowedIPs[0], LinkIndex: kiloIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, }, From a9fedecbfbd4738d2ff745f688fd9d0ccaa2ae5c Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 13 Feb 2026 21:09:35 +0100 Subject: [PATCH 2/2] mesh: set src for encapsulated tunl routes Signed-off-by: Andrei Kvapil --- .../kilo-azure-route-sync-deployment.yaml | 129 ++++++++++++++++++ pkg/mesh/routes.go | 7 +- pkg/mesh/routes_test.go | 13 ++ 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 manifests/kilo-azure-route-sync-deployment.yaml diff --git a/manifests/kilo-azure-route-sync-deployment.yaml b/manifests/kilo-azure-route-sync-deployment.yaml new file mode 100644 index 00000000..0d93f1dd --- /dev/null +++ b/manifests/kilo-azure-route-sync-deployment.yaml @@ -0,0 +1,129 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kilo-azure-route-sync + namespace: cozy-cluster-autoscaler-azure +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kilo-azure-route-sync +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kilo-azure-route-sync +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kilo-azure-route-sync +subjects: +- kind: ServiceAccount + name: kilo-azure-route-sync + namespace: cozy-cluster-autoscaler-azure +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kilo-azure-route-sync + namespace: cozy-cluster-autoscaler-azure +spec: + replicas: 1 + selector: + matchLabels: + app: kilo-azure-route-sync + template: + metadata: + labels: + app: kilo-azure-route-sync + spec: + serviceAccountName: kilo-azure-route-sync + containers: + - name: sync + image: mcr.microsoft.com/azure-cli:2.67.0 + imagePullPolicy: IfNotPresent + env: + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: cluster-autoscaler-azure-azure-cluster-autoscaler + key: ClientID + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: cluster-autoscaler-azure-azure-cluster-autoscaler + key: ClientSecret + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: cluster-autoscaler-azure-azure-cluster-autoscaler + key: TenantID + - name: AZURE_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + name: cluster-autoscaler-azure-azure-cluster-autoscaler + key: SubscriptionID + - name: AZURE_RESOURCE_GROUP + valueFrom: + secretKeyRef: + name: cluster-autoscaler-azure-azure-cluster-autoscaler + key: ResourceGroup + - name: AZURE_ROUTE_TABLE + value: kilo-routes-workers-serverscom + - name: AZURE_VNET_NAME + value: cozystack-vnet + - name: AZURE_SUBNET_NAME + value: workers-serverscom + - name: AZURE_ROUTES + value: to-serverscom=192.168.102.0/23 + command: ["/bin/sh","-ceu"] + args: + - | + az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" >/dev/null + az account set --subscription "$AZURE_SUBSCRIPTION_ID" + + az aks install-cli --install-location /usr/local/bin/kubectl >/dev/null + + sync_route() { + route_name="$1" + route_prefix="$2" + leader_ip="$3" + az network route-table route create -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \ + -n "$route_name" --address-prefix "$route_prefix" \ + --next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null || true + az network route-table route update -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \ + -n "$route_name" --address-prefix "$route_prefix" \ + --next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null + } + + sync_all_routes() { + leader_ip="$1" + IFS=',' + for entry in $AZURE_ROUTES; do + route_name="${entry%%=*}" + route_prefix="${entry#*=}" + [ -n "$route_name" ] && [ -n "$route_prefix" ] || continue + sync_route "$route_name" "$route_prefix" "$leader_ip" + done + unset IFS + } + + kubectl get node -w -l topology.kubernetes.io/zone=azure --no-headers \ + -o 'custom-columns=NAME:.metadata.name,LEADER:.metadata.annotations.kilo\.squat\.ai/leader,IP:.status.addresses[?(@.type=="InternalIP")].address' \ + | while read -r n leader ip; do + echo "$(date -Iseconds) event node=${n} leader=${leader} ip=${ip}" + [ "$leader" = "true" ] || continue + az network vnet subnet update \ + -g "$AZURE_RESOURCE_GROUP" \ + --vnet-name "$AZURE_VNET_NAME" \ + -n "$AZURE_SUBNET_NAME" \ + --route-table "$AZURE_ROUTE_TABLE" >/dev/null + + sync_all_routes "$ip" + + echo "$(date -Iseconds) synced routes to leader ${n} (${ip})" + done diff --git a/pkg/mesh/routes.go b/pkg/mesh/routes.go index 0c00be6f..c07456fb 100644 --- a/pkg/mesh/routes.go +++ b/pkg/mesh/routes.go @@ -76,6 +76,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface Flags: int(netlink.FLAG_ONLINK), Gw: segment.privateIPs[i], LinkIndex: tunlIface, + Src: t.privateIP.IP, Protocol: unix.RTPROT_STATIC, Table: kiloTableIndex, }) @@ -169,6 +170,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface Flags: int(netlink.FLAG_ONLINK), Gw: segment.privateIPs[i], LinkIndex: tunlIface, + Src: t.privateIP.IP, Protocol: unix.RTPROT_STATIC, Table: kiloTableIndex, }) @@ -316,8 +318,11 @@ func (t *Topology) PeerRoutes(name string, kiloIface int, additionalAllowedIPs [ } func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route { - if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) { + if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && subnet != nil && !subnet.Contains(route.Gw)) { route.LinkIndex = tunlIface + if subnet != nil && route.Src == nil { + route.Src = subnet.IP + } } return route } diff --git a/pkg/mesh/routes_test.go b/pkg/mesh/routes_test.go index aca1182a..edcf2a12 100644 --- a/pkg/mesh/routes_test.go +++ b/pkg/mesh/routes_test.go @@ -747,6 +747,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["c"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -754,6 +755,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["c"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["b"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, Table: kiloTableIndex, }, @@ -886,6 +888,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -893,6 +896,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -900,6 +904,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -907,6 +912,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -914,6 +920,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -921,6 +928,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, Table: kiloTableIndex, }, @@ -929,6 +937,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -936,6 +945,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -943,6 +953,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -950,6 +961,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, { @@ -957,6 +969,7 @@ func TestRoutes(t *testing.T) { Flags: int(netlink.FLAG_ONLINK), Gw: nodes["b"].InternalIP.IP, LinkIndex: tunlIface, + Src: nodes["c"].InternalIP.IP, Protocol: unix.RTPROT_STATIC, }, },