Skip to content

mesh: set preferred source for WireGuard routes #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kvaps merged 2 commits into from
Feb 14, 2026
+257 −1
Merged

mesh: set preferred source for WireGuard routes #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db4a4a5
mesh: set preferred source for WireGuard routes
kvaps Feb 13, 2026
a9fedec
mesh: set src for encapsulated tunl routes
kvaps Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 129 additions & 0 deletions manifests/kilo-azure-route-sync-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: kilo-azure-route-sync
namespace: cozy-cluster-autoscaler-azure
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kilo-azure-route-sync
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kilo-azure-route-sync
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kilo-azure-route-sync
subjects:
- kind: ServiceAccount
name: kilo-azure-route-sync
namespace: cozy-cluster-autoscaler-azure
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kilo-azure-route-sync
namespace: cozy-cluster-autoscaler-azure
spec:
replicas: 1
selector:
matchLabels:
app: kilo-azure-route-sync
template:
metadata:
labels:
app: kilo-azure-route-sync
spec:
serviceAccountName: kilo-azure-route-sync
containers:
- name: sync
image: mcr.microsoft.com/azure-cli:2.67.0
imagePullPolicy: IfNotPresent
env:
- name: AZURE_CLIENT_ID
valueFrom:
secretKeyRef:
name: cluster-autoscaler-azure-azure-cluster-autoscaler
key: ClientID
- name: AZURE_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: cluster-autoscaler-azure-azure-cluster-autoscaler
key: ClientSecret
- name: AZURE_TENANT_ID
valueFrom:
secretKeyRef:
name: cluster-autoscaler-azure-azure-cluster-autoscaler
key: TenantID
- name: AZURE_SUBSCRIPTION_ID
valueFrom:
secretKeyRef:
name: cluster-autoscaler-azure-azure-cluster-autoscaler
key: SubscriptionID
- name: AZURE_RESOURCE_GROUP
valueFrom:
secretKeyRef:
name: cluster-autoscaler-azure-azure-cluster-autoscaler
key: ResourceGroup
- name: AZURE_ROUTE_TABLE
value: kilo-routes-workers-serverscom
- name: AZURE_VNET_NAME
value: cozystack-vnet
- name: AZURE_SUBNET_NAME
value: workers-serverscom
- name: AZURE_ROUTES
value: to-serverscom=192.168.102.0/23
command: ["/bin/sh","-ceu"]
args:
- |
az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" >/dev/null
az account set --subscription "$AZURE_SUBSCRIPTION_ID"

az aks install-cli --install-location /usr/local/bin/kubectl >/dev/null

sync_route() {
route_name="$1"
route_prefix="$2"
leader_ip="$3"
az network route-table route create -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \
-n "$route_name" --address-prefix "$route_prefix" \
--next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null || true
az network route-table route update -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \
-n "$route_name" --address-prefix "$route_prefix" \
--next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null
}

sync_all_routes() {
leader_ip="$1"
IFS=','
for entry in $AZURE_ROUTES; do
route_name="${entry%%=*}"
route_prefix="${entry#*=}"
[ -n "$route_name" ] && [ -n "$route_prefix" ] || continue
sync_route "$route_name" "$route_prefix" "$leader_ip"
done
unset IFS
}

kubectl get node -w -l topology.kubernetes.io/zone=azure --no-headers \
-o 'custom-columns=NAME:.metadata.name,LEADER:.metadata.annotations.kilo\.squat\.ai/leader,IP:.status.addresses[?(@.type=="InternalIP")].address' \
| while read -r n leader ip; do
echo "$(date -Iseconds) event node=${n} leader=${leader} ip=${ip}"
[ "$leader" = "true" ] || continue
az network vnet subnet update \
-g "$AZURE_RESOURCE_GROUP" \
--vnet-name "$AZURE_VNET_NAME" \
-n "$AZURE_SUBNET_NAME" \
--route-table "$AZURE_ROUTE_TABLE" >/dev/null

sync_all_routes "$ip"

echo "$(date -Iseconds) synced routes to leader ${n} (${ip})"
done
19 changes: 18 additions & 1 deletion pkg/mesh/routes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
Flags: int(netlink.FLAG_ONLINK),
Gw: segment.privateIPs[i],
LinkIndex: tunlIface,
Src: t.privateIP.IP,
Protocol: unix.RTPROT_STATIC,
Table: kiloTableIndex,
})
Expand Down Expand Up @@ -135,6 +136,14 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
}
return routes, rules
}
// Compute the preferred source address for routes through the WireGuard interface.
// Without this, the kernel picks the WireGuard overlay IP (e.g. 100.66.0.x) as the
// source, which can cause issues in environments like Azure SDN where the overlay
// IP is unknown to the network fabric and reply packets cannot be routed back.
var src net.IP
if t.privateIP != nil {
src = t.privateIP.IP
}
for _, segment := range t.segments {
// Add routes for the current segment if local is true.
if segment.location == t.location {
Expand All @@ -161,6 +170,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
Flags: int(netlink.FLAG_ONLINK),
Gw: segment.privateIPs[i],
LinkIndex: tunlIface,
Src: t.privateIP.IP,
Protocol: unix.RTPROT_STATIC,
Table: kiloTableIndex,
})
Expand Down Expand Up @@ -190,6 +200,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
Flags: int(netlink.FLAG_ONLINK),
Gw: segment.wireGuardIP,
LinkIndex: kiloIface,
Src: src,
Protocol: unix.RTPROT_STATIC,
})
// Don't add routes through Kilo if the private IP
Expand All @@ -207,6 +218,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
Flags: int(netlink.FLAG_ONLINK),
Gw: segment.wireGuardIP,
LinkIndex: kiloIface,
Src: src,
Protocol: unix.RTPROT_STATIC,
})
}
Expand All @@ -218,6 +230,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
Flags: int(netlink.FLAG_ONLINK),
Gw: segment.wireGuardIP,
LinkIndex: kiloIface,
Src: src,
Protocol: unix.RTPROT_STATIC,
})
}
Expand All @@ -228,6 +241,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
routes = append(routes, &netlink.Route{
Dst: &peer.AllowedIPs[i],
LinkIndex: kiloIface,
Src: src,
Protocol: unix.RTPROT_STATIC,
})
}
Expand Down Expand Up @@ -304,8 +318,11 @@ func (t *Topology) PeerRoutes(name string, kiloIface int, additionalAllowedIPs [
}

func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && subnet != nil && !subnet.Contains(route.Gw)) {
route.LinkIndex = tunlIface
if subnet != nil && route.Src == nil {
route.Src = subnet.IP
}
}
return route
}
Expand Down
Loading