[BUG] Support OVN EIP,FIP and SNAT External subnet is not configured.

[InyongMa92]

Kube-OVN Version

1.13.2

Kubernetes Version

[root@vnode-103-150 ~]# kubectl version
Client Version: v1.30.8
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3

Operation-system/Kernel Version

[root@vnode-103-150 ~]# awk -F '=' '/PRETTY_NAME/ { print $2 }' /etc/os-release
"Rocky Linux 8.10 (Green Obsidian)"
[root@vnode-103-150 ~]# uname -r
4.18.0-513.5.1.el8_9.x86_64

Description

Support OVN EIP,FIP and SNAT(https://kubeovn.github.io/docs/v1.13.x/en/advance/ovn-eip-fip-snat/#support-ovn-eipfip-and-snat), I had tested external subnet as it decribed. But It doesn't work and I can not see any lrp in ovn-nbctl show vpc1.

I have 4 vms and installed kube-ovn. and trunk port is not available now. that's my enviornment right now. and I did what it described in the page.(https://kubeovn.github.io/docs/v1.13.x/en/advance/ovn-eip-fip-snat/#support-ovn-eipfip-and-snat)
1- label the nodes as gateway nodes
2- configure default external subnet
then attach extra external subnet. means the default subnet must be attached first to make it work

kubectl get provider-network's Ready status is false.

lrp of external network is not configured,

<Master Node , control plane> ifconfig
[root@vnode-103-150 ~]# ifconfig
6568e195e290_h: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::6cec:e5ff:fe85:a057 prefixlen 64 scopeid 0x20
ether 6e:ec:e5:85:a0:57 txqueuelen 1000 (Ethernet)
RX packets 159420 bytes 31593868 (30.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 160504 bytes 17329592 (16.5 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

85d65c431f48_h: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::34f4:d5ff:fefe:719c prefixlen 64 scopeid 0x20
ether 36:f4:d5:fe:71:9c txqueuelen 1000 (Ethernet)
RX packets 564788 bytes 48079375 (45.8 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 540773 bytes 474835711 (452.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

af013ac5e253_h: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::7802:cff:fed7:5060 prefixlen 64 scopeid 0x20
ether 7a:02:0c:d7:50:60 txqueuelen 1000 (Ethernet)
RX packets 160766 bytes 32051049 (30.5 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 161701 bytes 17517156 (16.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.9.103.150 netmask 255.255.0.0 broadcast 10.9.255.255
inet6 fe80::5054:ff:fe7d:5c43 prefixlen 64 scopeid 0x20
inet6 fd74:ca9b:3a09:868c:10:9:103:150 prefixlen 64 scopeid 0x0
ether 52:54:00:7d:5c:43 txqueuelen 1000 (Ethernet)
RX packets 47884887 bytes 5983849864 (5.5 GiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 2085808 bytes 1279615079 (1.1 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

genev_sys_6081: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 65000
inet6 fe80::8cf:8eff:fe89:5a37 prefixlen 64 scopeid 0x20
ether 0a:cf:8e:89:5a:37 txqueuelen 1000 (Ethernet)
RX packets 896129 bytes 68537858 (65.3 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1329450 bytes 951455334 (907.3 MiB)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10
loop txqueuelen 1000 (Local Loopback)
RX packets 14207754 bytes 8231615398 (7.6 GiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 14207754 bytes 8231615398 (7.6 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

mirror0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::479:fbff:fee2:74f4 prefixlen 64 scopeid 0x20
ether 06:79:fb:e2:74:f4 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 7 bytes 746 (746.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

ovn0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet 100.64.0.2 netmask 255.255.0.0 broadcast 100.64.255.255
inet6 fe80::8430:8bff:fe62:acef prefixlen 64 scopeid 0x20
ether 86:30:8b:62:ac:ef txqueuelen 1000 (Ethernet)
RX packets 1138516 bytes 90369150 (86.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1027750 bytes 1366747203 (1.2 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

<vnode-117-155 , worker node 1> ifconfig
[root@vnode-117-155 ~]# ifconfig
3b86896ada41_h: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::fc1d:eeff:fe40:2c63 prefixlen 64 scopeid 0x20
ether fe:1d:ee:40:2c:63 txqueuelen 1000 (Ethernet)
RX packets 567083 bytes 48457549 (46.2 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 536186 bytes 481988929 (459.6 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.9.117.155 netmask 255.255.0.0 broadcast 10.9.255.255
inet6 fe80::5054:ff:fec4:5c80 prefixlen 64 scopeid 0x20
inet6 fd74:ca9b:3a09:868c:10:9:117:155 prefixlen 64 scopeid 0x0
ether 52:54:00:c4:5c:80 txqueuelen 1000 (Ethernet)
RX packets 45907802 bytes 5409808888 (5.0 GiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1033178 bytes 130370637 (124.3 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

genev_sys_6081: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 65000
inet6 fe80::413:10ff:fec9:2f09 prefixlen 64 scopeid 0x20
ether 06:13:10:c9:2f:09 txqueuelen 1000 (Ethernet)
RX packets 535522 bytes 477591227 (455.4 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 566275 bytes 40639440 (38.7 MiB)
TX errors 0 dropped 7 overruns 0 carrier 0 collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10
loop txqueuelen 1000 (Local Loopback)
RX packets 298749 bytes 16329678 (15.5 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 298749 bytes 16329678 (15.5 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

mirror0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::b47b:c1ff:fe08:5a91 prefixlen 64 scopeid 0x20
ether b6:7b:c1:08:5a:91 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 7 bytes 746 (746.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

ovn0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet 100.64.0.3 netmask 255.255.0.0 broadcast 100.64.255.255
inet6 fe80::cce9:61ff:fe34:f977 prefixlen 64 scopeid 0x20
ether ce:e9:61:34:f9:77 txqueuelen 1000 (Ethernet)
RX packets 124451 bytes 6425886 (6.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 124641 bytes 8175416 (7.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

<vnode-117-156, worker node 2> ifconfig
[root@vnode-117-156 ~]# ifconfig
96fa86792bc0_h: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::3c20:2cff:fe72:7a43 prefixlen 64 scopeid 0x20
ether 3e:20:2c:72:7a:43 txqueuelen 1000 (Ethernet)
RX packets 555685 bytes 47517333 (45.3 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 526972 bytes 474234043 (452.2 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.9.117.156 netmask 255.255.0.0 broadcast 10.9.255.255
inet6 fe80::5054:ff:fe93:4e5a prefixlen 64 scopeid 0x20
inet6 fd74:ca9b:3a09:868c:10:9:117:156 prefixlen 64 scopeid 0x0
ether 52:54:00:93:4e:5a txqueuelen 1000 (Ethernet)
RX packets 45314333 bytes 5280708621 (4.9 GiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 796331 bytes 102904885 (98.1 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

genev_sys_6081: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 65000
inet6 fe80::1ca9:8dff:feb8:8a03 prefixlen 64 scopeid 0x20
ether 1e:a9:8d:b8:8a:03 txqueuelen 1000 (Ethernet)
RX packets 524648 bytes 466818755 (445.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 553386 bytes 39702813 (37.8 MiB)
TX errors 0 dropped 7 overruns 0 carrier 0 collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10
loop txqueuelen 1000 (Local Loopback)
RX packets 114797 bytes 6279708 (5.9 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 114797 bytes 6279708 (5.9 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

mirror0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet6 fe80::f4a6:81ff:fee3:63ec prefixlen 64 scopeid 0x20
ether f6:a6:81:e3:63:ec txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 7 bytes 746 (746.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

ovn0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
inet 100.64.0.5 netmask 255.255.0.0 broadcast 100.64.255.255
inet6 fe80::c40:35ff:feda:259f prefixlen 64 scopeid 0x20
ether be:e5:72:35:f3:65 txqueuelen 1000 (Ethernet)
RX packets 123347 bytes 6370296 (6.0 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 123543 bytes 8105834 (7.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

Steps To Reproduce

<provider network vlan, vlan-subnet config>
apiVersion: kubeovn.io/v1
kind: ProviderNetwork
metadata:
name: external204
spec:
defaultInterface: vlan

cat 02-vlan.yaml

apiVersion: kubeovn.io/v1
kind: Vlan
metadata:
name: vlan204
spec:
id: 204
provider: external204

cat 03-vlan-subnet.yaml

apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
name: external204
spec:
protocol: IPv4
cidrBlock: 10.5.204.0/24
gateway: 10.5.204.254
vlan: vlan204
excludeIps:

• 10.5.204.1..10.5.204.100

cat 00-centralized-external-gw-no-ip.yaml apiVersion: v1 kind: ConfigMap metadata: name: ovn-external-gw-config namespace: kube-system data: enable-external-gw: "true" external-gw-nodes: "vnode-117-155,vnode-117-156” type: "centralized" external-gw-nic: "vlan" external-gw-addr: "10.5.204.254/24"

#label in worker node
kubectl label nodes vnode-117-155 vnode-117-156 ovn.kubernetes.io/external-gw=true

<ns, vpc, vpc-subnet config>

cat 00-ns.yml

apiVersion: v1
kind: Namespace
metadata:
name: vpc1

cat 01-vpc-ecmp-enable-external-bfd.yml

kind: Vpc
apiVersion: kubeovn.io/v1
metadata:
name: vpc1
spec:
namespaces:

• vpc1
  enableExternal: true

vpc enableExternal will automatically create an lrp association to the public network specified above

cat 02-subnet.yml

apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
name: vpc1-subnet1
spec:
cidrBlock: 192.168.0.0/24
default: false
disableGatewayCheck: false
disableInterConnection: true
enableEcmp: true
gatewayNode: ""
gatewayType: distributed
#gatewayType: centralized
natOutgoing: false
private: false
protocol: IPv4
provider: ovn
vpc: vpc1
namespaces:

• vpc1

Current Behavior

ovn-nbctl show vpc1 can't see external subnet lrp

Expected Behavior

well configured external subnet

[dosubot]

Hello, @InyongMa92! I'm here to help with bug fixes, answer questions, and guide you in becoming a contributor. Let's tackle this challenge together!

To address the issue of the external subnet not being configured correctly in Kube-OVN, here are some steps and checks you can perform:

1. Check Provider Network Readiness: The 'Ready' status of your provider network is false, which indicates potential issues. Common reasons include:

   • Nodes listed in pn.Spec.ExcludeNodes are excluded.
   • Nodes lack the pnReadyAnnotation label set to "true".
   • Missing kube-ovn-cni pod on nodes.
   • Errors in kube-ovn-cni pod annotations ^[1].

2. Verify Node Labeling: Ensure that the nodes intended to act as external gateways are correctly labeled. You have labeled vnode-117-155 and vnode-117-156 with ovn.kubernetes.io/external-gw=true, which is correct. Ensure these nodes are not excluded from the provider network.

3. External Gateway Configuration: Double-check the ovn-external-gw-config ConfigMap. Ensure that:

   • enable-external-gw is set to "true".
   • external-gw-nodes lists the correct nodes.
   • external-gw-nic is correctly set to the NIC used for external traffic. Ensure this NIC is not already attached to another bridge ^[2].

4. VPC Configuration: Ensure that the VPC configuration has enableExternal: true, which should automatically create an LRP association with the specified public network. Verify that the external subnet is correctly defined and associated with the VPC ^[3].

5. Logical Router Port (LRP) Setup: If the LRP for the external network is not appearing, ensure that the logical router port creation process is correctly triggered. This involves checking the OVN Northbound database interactions and ensuring that the necessary logical router ports are created and associated with the external subnet ^[4].

6. Network Attachment Definition: Ensure that the external subnet's provider network is correctly set up and that the Network Attachment Definition is properly configured to facilitate the external network setup ^[3].

By following these steps, you should be able to identify and resolve the configuration issues preventing the external subnet from being correctly set up in Kube-OVN.

To continue talking to Dosu, mention @dosu.

---

^{Help Dosu learn! Give it feedback: Great Response | Irrelevant Answer | Incorrect Sources | Too Verbose | Hallucination | Other}

[bobz965]

problem： I had tested external subnet as it decribed. But It doesn't work and I can not see any lrp in ovn-nbctl show vpc1.

please show the the image which contains nbctl show，and kubeclt get ovn-eip ovn-fip

[InyongMa92]

Hello, I post what you request info.
external204 is external network(10.5.204.0/24) and vpc1-subnet1(192.168.0.0/24) is internal network

[root@vnode-103-150 ~]# kubectl get provider-network
NAME DEFAULTINTERFACE READY
external204 vlan false
extra vlan false

#ovn-nbctl show vpc1
$ ovn-nbctl show vpc1
router d0434b76-3283-4333-816f-7a89678a3b3b (vpc1)
port vpc1-vpc1-subnet1
mac: "22:9e:5c:c7:7e:d1"
networks: ["192.168.0.1/24"]
nat 145337f4-9670-42c7-b90b-1d69f8940d4c
external ip: "10.5.204.101"
logical ip: "192.168.0.2"
type: "dnat_and_snat"

#ovn-nbctl show
$ ovn-nbctl show
switch 8107ef34-98f2-41d5-b98b-4de9fd82f906 (external204)
port localnet.external204
type: localnet
tag: 204
addswitch 8107ef34-98f2-41d5-b98b-4de9fd82f906 (external204)
port localnet.external204
type: localnet
tag: 204
addresses: ["unknown"]
switch 43becf51-c828-416d-9707-42627180a05b (join)
port node-vnode-117-155
addresses: ["ce:e9:61:34:f9:77 100.64.0.3"]
port join-ovn-cluster
type: router
router-port: ovn-cluster-join
port node-vnode-117-156
addresses: ["be:e5:72:35:f3:65 100.64.0.5"]
port node-vnode-103-150
addresses: ["86:30:8b:62:ac:ef 100.64.0.2"]
switch ee3fbb21-7e3b-424b-8e45-88deefa51ebb (vpc1-subnet1)
port vpc1-pod-3.vpc1
addresses: ["e6:09:8d:53:61:ef 192.168.0.4"]
port vpc1-pod-2.vpc1
addresses: ["2e:55:58:21:29:5b 192.168.0.3"]
port vpc1-subnet1-vpc1
type: router
router-port: vpc1-vpc1-subnet1
port vpc1-pod-1.vpc1
addresses: ["a6:31:86:62:74:17 192.168.0.2"]
switch 36626eba-1359-4fb3-aadc-024b194f783d (ovn-default)
port virt-handler-j5sz2.kubevirt
addresses: ["d6:99:cd:16:57:ff 10.16.0.23"]
port kube-ovn-pinger-xhxqv.kube-system
addresses: ["ce:f5:18:b8:ff:6f 10.16.0.8"]
port virt-operator-5b5f954844-gs5tt.kubevirt
addresses: ["e2:17:3f:e4:c4:5b 10.16.0.15"]
port kube-ovn-pinger-bql9p.kube-system
addresses: ["fa:ae:77:f7:5b:d5 10.16.0.4"]
port coredns-55cb58b774-dnlfw.kube-system
addresses: ["d2:ae:1a:9b:52:fe 10.16.0.2"]
port coredns-55cb58b774-l5tqc.kube-system
addresses: ["7e:d0:80:27:22:f6 10.16.0.3"]
port test-vm.default
addresses: ["5e:33:a8:c8:2b:25 10.16.0.25"]
port virt-api-768454998c-fdzvx.kubevirt
addresses: ["86:4d:5d:cb:88:c8 10.16.0.19"]
port virt-api-768454998c-cnpx7.kubevirt
addresses: ["0e:13:b4:54:1d:f6 10.16.0.20"]
port kube-ovn-pinger-x76z7.kube-system
addresses: ["5a:39:34:f0:96:03 10.16.0.7"]
port virt-controller-7466b4d5f-phw52.kubevirt
addresses: ["12:6f:06:61:b4:a4 10.16.0.22"]
port virt-operator-5b5f954844-fvlf7.kubevirt
addresses: ["6e:85:52:34:51:34 10.16.0.17"]
port sg-test-pod.default
addresses: ["26:e9:71:c7:b5:04 10.16.0.14"]
port virt-handler-cxfxd.kubevirt
addresses: ["ca:e4:14:ea:a0:7e 10.16.0.24"]
port virt-controller-7466b4d5f-shlng.kubevirt
addresses: ["ae:e1:ff:00:8e:ee 10.16.0.21"]
port ovn-default-ovn-cluster
type: router
router-port: ovn-cluster-ovn-default
router dbe03529-f83e-4433-ba07-8bf72e691d7f (ovn-cluster)
port ovn-cluster-join
mac: "32:f4:a8:4a:5c:cc"
networks: ["100.64.0.1/16"]
port ovn-cluster-ovn-default
mac: "6a:61:19:d8:36:ba"
networks: ["10.16.0.1/16"]
router d0434b76-3283-4333-816f-7a89678a3b3b (vpc1)
port vpc1-vpc1-subnet1
mac: "22:9e:5c:c7:7e:d1"
networks: ["192.168.0.1/24"]
nat 145337f4-9670-42c7-b90b-1d69f8940d4c
external ip: "10.5.204.101"
logical ip: "192.168.0.2"
type: "dnat_and_snat"

#kubectl get ovn-fip
[root@vnode-103-150 ~]# kubectl get ovn-fip
NAME VPC V4EIP V6EIP V4IP V6IP READY IPTYPE IPNAME
eip-static vpc1 10.5.204.101 192.168.0.2 true vpc1-pod-1.vpc1

#kubectl get ovn-eip
[root@vnode-103-150 ~]# kubectl get ovn-eip
NAME V4IP V6IP MAC TYPE NAT READY EXTERNALSUBNET
eip-static 10.5.204.101 6e:8e:28:2a:75:d7 nat true external204

[bobz965]

please check the kubeovn controller log about vpc1 （grep vpc1）

[bobz965]

do you apply the configmap about enable eip snat ?