Skip to content

Commit b7f58db

Browse files
struanbstruanb
committed
Support for calling iptables' --wait and --wait-seconds options
2 parents 0676caf + fb9fbe9 commit b7f58db

File tree

2 files changed

+48
-34
lines changed

2 files changed

+48
-34
lines changed

‎README.md

+5
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ N.B. Following production testing, for performance reasons the daemon also perfo
6969
7070
> N.B. If you do not specify `--services` and `--tcp-ports` / `--udp-ports` then *ALL* service containers with an ingress network interface will be reconfigured and your access to non-service containers publishing ports might freeze.
7171
72+
> **WARNING:** The use of `--iptables-wait` or `--iptables-wait <n>` is strongly advised if your systems run `firewalld` or another process that performs high-frequency access to iptables. In such scenarios the xtables lock could be held, inhibiting DIRD making successful calls to iptables and producing the error *"Another app is currently holding the xtables lock"*. The use of these options should be harmless in other cases.
73+
7274
### Running the daemon
7375

7476
Having prepared your command line options (collectively, `[OPTIONS]`), run the following **as root** on **_each and every one_** of your load-balancer and/or service container nodes:
@@ -100,6 +102,9 @@ Usage: ./docker-ingress-routing-daemon [--install [OPTIONS] | --uninstall | --he
100102
--no-performance - disable performance optimisations
101103
--indexed-ids - use sequential ids for load balancers
102104
(forced where ingress subnet larger than /24)
105+
106+
--iptables-wait - pass '--iptables-wait' option to iptables
107+
--iptables-wait-seconds <n> - pass '--iptables-wait <n>' option to iptables
103108
104109
(services, ports and IPs may be comma or space-separated or may be specified
105110
multiple times)

‎docker-ingress-routing-daemon

+43-34
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
#!/bin/bash
22

3-
VERSION=4.1.2
3+
VERSION=4.2.0
4+
IPTABLES_OPTS=()
45

5-
# Ingress Routing Daemon v4.1.2
6-
# Copyright © 2020-2021 Struan Bartlett
6+
# Ingress Routing Daemon v4.2.0
7+
# Copyright © 2020-2023 Struan Bartlett
78
# ----------------------------------------------------------------------
89
# Permission is hereby granted, free of charge, to any person
910
# obtaining a copy of this software and associated documentation files
@@ -79,7 +80,7 @@ route_ingress() {
7980
fi
8081

8182
local IPTABLE_COMMENT="docker-ingress-routing-daemon"
82-
if nsenter -n -t $NID iptables -t mangle -C OUTPUT -m comment --comment "$IPTABLE_COMMENT" 2>/dev/null; then
83+
if nsenter -n -t $NID iptables "${IPTABLES_OPTS[@]}" -t mangle -C OUTPUT -m comment --comment "$IPTABLE_COMMENT" 2>/dev/null; then
8384
log "Detected container for service '$SERVICE', with ID '$ID' and NID '$NID': mangle table already configured, so skipping."
8485
return
8586
fi
@@ -92,9 +93,9 @@ route_ingress() {
9293
# TOS byte has been set by the load balancer, then none will be restored and legacy routing rules will apply.
9394
# - See https://github.com/newsnowlabs/docker-ingress-routing-daemon/issues/11
9495
log "- Adding container mangle table iptables rules"
95-
nsenter -n -t $NID iptables -t mangle -A OUTPUT -m comment --comment "$IPTABLE_COMMENT"
96-
nsenter -n -t $NID iptables -t mangle -A OUTPUT -p udp -j CONNMARK --restore-mark
97-
nsenter -n -t $NID iptables -t mangle -A OUTPUT -p tcp -j CONNMARK --restore-mark
96+
nsenter -n -t $NID iptables "${IPTABLES_OPTS[@]}" -t mangle -A OUTPUT -m comment --comment "$IPTABLE_COMMENT"
97+
nsenter -n -t $NID iptables "${IPTABLES_OPTS[@]}" -t mangle -A OUTPUT -p udp -j CONNMARK --restore-mark
98+
nsenter -n -t $NID iptables "${IPTABLES_OPTS[@]}" -t mangle -A OUTPUT -p tcp -j CONNMARK --restore-mark
9899

99100
# 3.1 Enable 'loose' rp_filter mode on interface $CIF (and 'all' as required by kernel
100101
# see https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt)
@@ -114,7 +115,7 @@ route_ingress() {
114115
log "- Adding container policy routing/firewall rules for load-balancer #$NODE_ID with IP $NODE_IP"
115116

116117
# 2. Map the TOS value on any incoming packets to a connection mark, using the same value.
117-
nsenter -n -t $NID iptables -t mangle -A PREROUTING -m tos --tos $NODE_ID/0xff -j CONNMARK --set-xmark $NODE_ID/0xffffffff
118+
nsenter -n -t $NID iptables "${IPTABLES_OPTS[@]}" -t mangle -A PREROUTING -m tos --tos $NODE_ID/0xff -j CONNMARK --set-xmark $NODE_ID/0xffffffff
118119

119120
# 4. Select the correct routing table to use, according to the firewall mark on the outgoing packet.
120121
nsenter -n -t $NID ip rule add from $INGRESS_SUBNET fwmark $NODE_ID lookup $NODE_ID prio 32700
@@ -140,6 +141,9 @@ usage() {
140141
echo " --preexisting - optionally install rules where needed" >&2
141142
echo " on preexisting containers (recommended)" >&2
142143
echo >&2
144+
echo " --iptables-wait - pass '--wait' option to iptables" >&2
145+
echo " --iptables-wait-seconds <n> - pass '--wait <n>' option to iptables" >&2
146+
echo >&2
143147
echo " --no-performance - disable performance optimisations" >&2
144148
echo " --indexed-ids - use sequential ids for load balancers" >&2
145149
echo " (forced where ingress subnet larger than /24)" >&2
@@ -177,6 +181,8 @@ do
177181
--no-performance) shift; PERFORMANCE=0; continue; ;;
178182
--indexed-ids) shift; INDEXED_IDS=1; continue; ;;
179183
--preexisting) shift; PREEXISTING=1; continue; ;;
184+
--iptables-wait) shift; IPTABLES_WAIT=1; continue; ;;
185+
--iptables-wait-seconds) shift; IPTABLES_WAIT=1; IPTABLES_WAIT_SECONDS="$1"; shift; continue; ;;
180186

181187
-h|--help) usage; ;;
182188
'') break; ;;
@@ -188,6 +194,9 @@ done
188194
# Display usage, unless --install or --uninstall
189195
[ -z "$INSTALL" ] && usage
190196

197+
# Add '--iptables-wait' or '--iptables-wait $IPTABLES_WAIT_SECONDS' to iptables options
198+
[ -n "$IPTABLES_WAIT" ] && IPTABLES_OPTS+=(--wait $IPTABLES_WAIT_SECONDS)
199+
191200
# Convert arrays to comma-separated strings
192201
TCPServicePortString=$(echo ${TCP_PORTS[@]} | tr ' ' ',')
193202
UDPServicePortString=$(echo ${UDP_PORTS[@]} | tr ' ' ',')
@@ -215,31 +224,31 @@ fi
215224
# Delete any relevant preexisting rules.
216225
log "Cleaning up any stale load-balancer rules ..."
217226

218-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t nat -S | \
227+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t nat -S | \
219228
grep -- '-m ipvs --ipvs -j ACCEPT' | \
220229
sed -r 's/^-A /-D /' | \
221230
while read RULE; \
222231
do
223-
log "- Deleting old rule: iptables -t nat $RULE"
224-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t nat $RULE
232+
log "- Deleting old rule: iptables "${IPTABLES_OPTS[@]}" -t nat $RULE"
233+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t nat $RULE
225234
done
226235

227-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t mangle -S | \
236+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t mangle -S | \
228237
grep -- '-j TOS --set-tos' | \
229238
sed -r 's/^-A /-D /' | \
230239
while read RULE; \
231240
do
232-
log "- Deleting old rule: iptables -t mangle $RULE"
233-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t mangle $RULE
241+
log "- Deleting old rule: iptables "${IPTABLES_OPTS[@]}" -t mangle $RULE"
242+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t mangle $RULE
234243
done
235244

236-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t raw -S | \
245+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t raw -S | \
237246
grep -- '-j CT --notrack' | \
238247
sed -r 's/^-A /-D /' | \
239248
while read RULE; \
240249
do
241-
log "- Deleting old rule: iptables -t raw $RULE"
242-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t raw $RULE
250+
log "- Deleting old rule: iptables "${IPTABLES_OPTS[@]}" -t raw $RULE"
251+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t raw $RULE
243252
done
244253

245254
if [ "$INSTALL" = "0" ]; then
@@ -309,39 +318,39 @@ if [ -n "$NODE_ID" ]; then
309318

310319
# Add a rule ahead of the ingress network SNAT rule, that will cause the SNAT rule to be skipped.
311320
if [ -z "$TCPServicePortString" ] && [ -z "$UDPServicePortString" ]; then
312-
log "- Adding ingress_sbox iptables nat rule: iptables -t nat -I POSTROUTING -d $INGRESS_SUBNET -m ipvs --ipvs -j ACCEPT"
313-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t nat -I POSTROUTING -d $INGRESS_SUBNET -m ipvs --ipvs -j ACCEPT
321+
log "- Adding ingress_sbox iptables nat rule: iptables "${IPTABLES_OPTS[@]}" -t nat -I POSTROUTING -d $INGRESS_SUBNET -m ipvs --ipvs -j ACCEPT"
322+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t nat -I POSTROUTING -d $INGRESS_SUBNET -m ipvs --ipvs -j ACCEPT
314323

315324
# 1. Set TOS to NODE_ID in all outgoing packets to INGRESS_SUBNET
316-
log "- Adding ingress_sbox iptables mangle rule: iptables -t mangle -A POSTROUTING -d $INGRESS_SUBNET -j TOS --set-tos $NODE_ID/0xff"
317-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t mangle -A POSTROUTING -d $INGRESS_SUBNET -j TOS --set-tos $NODE_ID/0xff
325+
log "- Adding ingress_sbox iptables mangle rule: iptables "${IPTABLES_OPTS[@]}" -t mangle -A POSTROUTING -d $INGRESS_SUBNET -j TOS --set-tos $NODE_ID/0xff"
326+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t mangle -A POSTROUTING -d $INGRESS_SUBNET -j TOS --set-tos $NODE_ID/0xff
318327

319-
log "- Adding ingress_sbox connection tracking disable rule: iptables -t raw -I PREROUTING -j CT --notrack"
320-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t raw -I PREROUTING -j CT --notrack
328+
log "- Adding ingress_sbox connection tracking disable rule: iptables "${IPTABLES_OPTS[@]}" -t raw -I PREROUTING -j CT --notrack"
329+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t raw -I PREROUTING -j CT --notrack
321330
else
322331

323332
if [ -n "$TCPServicePortString" ]; then
324-
log "- Adding ingress_sbox iptables nat rule: iptables -t nat -I POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -m ipvs --ipvs -j ACCEPT"
325-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t nat -I POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -m ipvs --ipvs -j ACCEPT
333+
log "- Adding ingress_sbox iptables nat rule: iptables "${IPTABLES_OPTS[@]}" -t nat -I POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -m ipvs --ipvs -j ACCEPT"
334+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t nat -I POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -m ipvs --ipvs -j ACCEPT
326335

327336
# 1. Set TOS to NODE_ID in all outgoing packets to INGRESS_SUBNET
328-
log "- Adding ingress_sbox iptables mangle rule: iptables -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -j TOS --set-tos $NODE_ID/0xff"
329-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -j TOS --set-tos $NODE_ID/0xff
337+
log "- Adding ingress_sbox iptables mangle rule: iptables "${IPTABLES_OPTS[@]}" -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -j TOS --set-tos $NODE_ID/0xff"
338+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p tcp -m multiport --dports $TCPServicePortString -j TOS --set-tos $NODE_ID/0xff
330339

331-
log "- Adding ingress_sbox connection tracking disable rule: iptables -t raw -I PREROUTING -p tcp -m multiport --dports $TCPServicePortString -j CT --notrack"
332-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t raw -I PREROUTING -p tcp -m multiport --dports $TCPServicePortString -j CT --notrack
340+
log "- Adding ingress_sbox connection tracking disable rule: iptables "${IPTABLES_OPTS[@]}" -t raw -I PREROUTING -p tcp -m multiport --dports $TCPServicePortString -j CT --notrack"
341+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t raw -I PREROUTING -p tcp -m multiport --dports $TCPServicePortString -j CT --notrack
333342
fi
334343

335344
if [ -n "$UDPServicePortString" ]; then
336-
log "- Adding ingress_sbox iptables nat rule: iptables -t nat -I POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -m ipvs --ipvs -j ACCEPT"
337-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t nat -I POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -m ipvs --ipvs -j ACCEPT
345+
log "- Adding ingress_sbox iptables nat rule: iptables "${IPTABLES_OPTS[@]}" -t nat -I POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -m ipvs --ipvs -j ACCEPT"
346+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t nat -I POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -m ipvs --ipvs -j ACCEPT
338347

339348
# 1. Set TOS to NODE_ID in all outgoing packets to INGRESS_SUBNET
340-
log "- Adding ingress_sbox iptables mangle rule: iptables -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -j TOS --set-tos $NODE_ID/0xff"
341-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -j TOS --set-tos $NODE_ID/0xff
349+
log "- Adding ingress_sbox iptables mangle rule: iptables "${IPTABLES_OPTS[@]}" -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -j TOS --set-tos $NODE_ID/0xff"
350+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t mangle -A POSTROUTING -d $INGRESS_SUBNET -p udp -m multiport --dports $UDPServicePortString -j TOS --set-tos $NODE_ID/0xff
342351

343352
log "- Adding ingress_sbox connection tracking disable rule: iptables -p udp -m multiport --dports $UDPServicePortString -j CT --notrack"
344-
nsenter --net=/var/run/docker/netns/ingress_sbox iptables -t raw -I PREROUTING -p udp -m multiport --dports $UDPServicePortString -j CT --notrack
353+
nsenter --net=/var/run/docker/netns/ingress_sbox iptables "${IPTABLES_OPTS[@]}" -t raw -I PREROUTING -p udp -m multiport --dports $UDPServicePortString -j CT --notrack
345354
fi
346355

347356
fi

0 commit comments

Comments
 (0)