2026-07-29T08:48:10.603591623Z + . /ovnkube-lib/ovnkube-lib.sh 2026-07-29T08:48:10.603675880Z ++ set -x 2026-07-29T08:48:10.603675880Z ++ K8S_NODE=ip-10-0-133-190.ec2.internal 2026-07-29T08:48:10.603675880Z ++ [[ -n ip-10-0-133-190.ec2.internal ]] 2026-07-29T08:48:10.603675880Z ++ [[ -f /env/ip-10-0-133-190.ec2.internal ]] 2026-07-29T08:48:10.603675880Z ++ northd_pidfile=/var/run/ovn/ovn-northd.pid 2026-07-29T08:48:10.603675880Z ++ controller_pidfile=/var/run/ovn/ovn-controller.pid 2026-07-29T08:48:10.603675880Z ++ controller_logfile=/var/log/ovn/acl-audit-log.log 2026-07-29T08:48:10.603675880Z ++ vswitch_dbsock=/var/run/openvswitch/db.sock 2026-07-29T08:48:10.603675880Z ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid 2026-07-29T08:48:10.603686572Z ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock 2026-07-29T08:48:10.603686572Z ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl 2026-07-29T08:48:10.603686572Z ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid 2026-07-29T08:48:10.603695939Z ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock 2026-07-29T08:48:10.603702641Z ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl 2026-07-29T08:48:10.604991200Z + start-ovnkube-node 4 29103 29105 2026-07-29T08:48:10.605045899Z + local log_level=4 2026-07-29T08:48:10.605053494Z + local metrics_port=29103 2026-07-29T08:48:10.605053494Z + local ovn_metrics_port=29105 2026-07-29T08:48:10.605060484Z + ovn_advertised_udn_isolation_mode_flag= 2026-07-29T08:48:10.605067037Z + openflow_probe_flag= 2026-07-29T08:48:10.605067037Z + [[ 3 -ne 3 ]] 2026-07-29T08:48:10.605075566Z + ovs-vsctl br-exists br-ex 2026-07-29T08:48:10.612363970Z + wait-and-add-garp-drop-flow br-ex 2026-07-29T08:48:10.612363970Z + cni-bin-copy 2026-07-29T08:48:10.612414610Z + . /host/etc/os-release 2026-07-29T08:48:10.612424044Z + local bridge=br-ex 2026-07-29T08:48:10.612442058Z + local max_retries=90 2026-07-29T08:48:10.612448886Z + local retries=0 2026-07-29T08:48:10.612464267Z + [[ 0 -lt 90 ]] 2026-07-29T08:48:10.612464267Z ++ NAME='Red Hat Enterprise Linux CoreOS' 2026-07-29T08:48:10.612471850Z ++ VERSION='9.6.20260717-0 (Plow)' 2026-07-29T08:48:10.612471850Z + add_garp_drop_flow br-ex 2026-07-29T08:48:10.612478499Z ++ ID=rhel 2026-07-29T08:48:10.612485125Z ++ ID_LIKE=fedora 2026-07-29T08:48:10.612491931Z ++ VERSION_ID=9.6 2026-07-29T08:48:10.612491931Z ++ PLATFORM_ID=platform:el9 2026-07-29T08:48:10.612498520Z + local bridge=br-ex 2026-07-29T08:48:10.612498520Z ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260717-0 (Plow)' 2026-07-29T08:48:10.612505301Z + local cookie=0x0305 2026-07-29T08:48:10.612505301Z ++ ANSI_COLOR='0;31' 2026-07-29T08:48:10.612505301Z ++ LOGO=fedora-logo-icon 2026-07-29T08:48:10.612512855Z + local priority=499 2026-07-29T08:48:10.612512855Z ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos 2026-07-29T08:48:10.612519499Z ++ HOME_URL=https://www.redhat.com/ 2026-07-29T08:48:10.612525999Z ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 2026-07-29T08:48:10.612532602Z ++ BUG_REPORT_URL=https://issues.redhat.com/ 2026-07-29T08:48:10.612532602Z ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' 2026-07-29T08:48:10.612539312Z ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 2026-07-29T08:48:10.612547392Z ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' 2026-07-29T08:48:10.612553921Z ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 2026-07-29T08:48:10.612560331Z ++ OSTREE_VERSION=9.6.20260717-0 2026-07-29T08:48:10.612560331Z ++ VARIANT=CoreOS 2026-07-29T08:48:10.612574964Z ++ VARIANT_ID=coreos 2026-07-29T08:48:10.612574964Z ++ OPENSHIFT_VERSION=4.21 2026-07-29T08:48:10.612574964Z + rhelmajor= 2026-07-29T08:48:10.612582252Z + case "${ID}" in 2026-07-29T08:48:10.612830445Z ++ ovs-vsctl list-ports br-ex 2026-07-29T08:48:10.613165349Z ++ echo 9.6 2026-07-29T08:48:10.613287841Z ++ cut -f 1 -d . 2026-07-29T08:48:10.614851672Z + rhelmajor=9 2026-07-29T08:48:10.614869963Z + sourcedir=/usr/libexec/cni/ 2026-07-29T08:48:10.614869963Z + case "${rhelmajor}" in 2026-07-29T08:48:10.614869963Z + sourcedir=/usr/libexec/cni/rhel9 2026-07-29T08:48:10.614909129Z + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ 2026-07-29T08:48:10.619171701Z + for port_name in $(ovs-vsctl list-ports "$bridge") 2026-07-29T08:48:10.619171701Z + [[ ens5 == *to-br-int ]] 2026-07-29T08:48:10.619192615Z + return 1 2026-07-29T08:48:10.619199925Z + (( retries += 1 )) 2026-07-29T08:48:10.619208097Z + [[ 1 -eq 15 ]] 2026-07-29T08:48:10.619214462Z + sleep 2 2026-07-29T08:48:10.672976815Z ++ date '+%m%d %H:%M:%S.%N' 2026-07-29T08:48:10.674551229Z + echo 'I0729 08:48:10.674329730 - disable conntrack on geneve port' 2026-07-29T08:48:10.674583147Z I0729 08:48:10.674329730 - disable conntrack on geneve port 2026-07-29T08:48:10.674594087Z + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK 2026-07-29T08:48:10.713630014Z + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK 2026-07-29T08:48:10.715181426Z + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK 2026-07-29T08:48:10.716790477Z + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK 2026-07-29T08:48:10.718487799Z ++ date '+%m%d %H:%M:%S.%N' 2026-07-29T08:48:10.719815580Z + echo 'I0729 08:48:10.719626990 - starting ovnkube-node' 2026-07-29T08:48:10.719833369Z I0729 08:48:10.719626990 - starting ovnkube-node 2026-07-29T08:48:10.719839633Z + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' 2026-07-29T08:48:10.719860787Z + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-133-190.ec2.internal' 2026-07-29T08:48:10.719871026Z + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true 2026-07-29T08:48:10.719871026Z + gateway_interface=br-ex 2026-07-29T08:48:10.719880458Z + enable_multicast_flag=--enable-multicast 2026-07-29T08:48:10.719901374Z + OVN_NODE_MODE=full 2026-07-29T08:48:10.719925805Z + '[' full == dpu-host ']' 2026-07-29T08:48:10.719946299Z + '[' shared == shared ']' 2026-07-29T08:48:10.719956169Z + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' 2026-07-29T08:48:10.719965642Z + export_network_flows_flags= 2026-07-29T08:48:10.719977043Z + [[ -n '' ]] 2026-07-29T08:48:10.719988663Z + [[ -n '' ]] 2026-07-29T08:48:10.719997861Z + [[ -n '' ]] 2026-07-29T08:48:10.719997861Z + [[ -n '' ]] 2026-07-29T08:48:10.720011008Z + [[ -n '' ]] 2026-07-29T08:48:10.720020423Z + [[ -n '' ]] 2026-07-29T08:48:10.720020423Z + gw_interface_flag= 2026-07-29T08:48:10.720039713Z + '[' -d /sys/class/net/br-ex1 ']' 2026-07-29T08:48:10.720063062Z + node_mgmt_port_netdev_flags= 2026-07-29T08:48:10.720073120Z + [[ -n '' ]] 2026-07-29T08:48:10.720082960Z + [[ -n '' ]] 2026-07-29T08:48:10.720082960Z + multi_network_enabled_flag= 2026-07-29T08:48:10.720099238Z + [[ true == \t\r\u\e ]] 2026-07-29T08:48:10.720099238Z + [[ full != \d\p\u\-\h\o\s\t ]] 2026-07-29T08:48:10.720109091Z + multi_network_enabled_flag=--enable-multi-network 2026-07-29T08:48:10.720109091Z + network_segmentation_enabled_flag= 2026-07-29T08:48:10.720118689Z + [[ true == \t\r\u\e ]] 2026-07-29T08:48:10.720127996Z + [[ full != \d\p\u\-\h\o\s\t ]] 2026-07-29T08:48:10.720127996Z + multi_network_enabled_flag=--enable-multi-network 2026-07-29T08:48:10.720137871Z + network_segmentation_enabled_flag=--enable-network-segmentation 2026-07-29T08:48:10.720147848Z + route_advertisements_enable_flag= 2026-07-29T08:48:10.720147848Z + [[ false == \t\r\u\e ]] 2026-07-29T08:48:10.720157213Z + preconfigured_udn_addresses_enable_flag= 2026-07-29T08:48:10.720166442Z + [[ true == \t\r\u\e ]] 2026-07-29T08:48:10.720166442Z + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses 2026-07-29T08:48:10.720176113Z + network_observability_enabled_flag= 2026-07-29T08:48:10.720176113Z + [[ false == \t\r\u\e ]] 2026-07-29T08:48:10.720185893Z + multi_network_policy_enabled_flag= 2026-07-29T08:48:10.720195070Z + [[ false == \t\r\u\e ]] 2026-07-29T08:48:10.720195070Z + admin_network_policy_enabled_flag= 2026-07-29T08:48:10.720204526Z + [[ true == \t\r\u\e ]] 2026-07-29T08:48:10.720204526Z + [[ full != \d\p\u\-\h\o\s\t ]] 2026-07-29T08:48:10.720213923Z + admin_network_policy_enabled_flag=--enable-admin-network-policy 2026-07-29T08:48:10.720213923Z + dns_name_resolver_enabled_flag= 2026-07-29T08:48:10.720223106Z + [[ false == \t\r\u\e ]] 2026-07-29T08:48:10.720232144Z + ip_forwarding_flag= 2026-07-29T08:48:10.720241226Z + '[' '' == Global ']' 2026-07-29T08:48:10.720241226Z + ip_forwarding_flag=--disable-forwarding 2026-07-29T08:48:10.720252426Z + sysctl -w net.ipv4.ip_forward=0 2026-07-29T08:48:10.731056972Z net.ipv4.ip_forward = 0 2026-07-29T08:48:10.731152149Z + sysctl -w net.ipv6.conf.all.forwarding=0 2026-07-29T08:48:10.731860071Z net.ipv6.conf.all.forwarding = 0 2026-07-29T08:48:10.731966209Z + [[ '' != '' ]] 2026-07-29T08:48:10.731981963Z + [[ '' != '' ]] 2026-07-29T08:48:10.731981963Z + NETWORK_NODE_IDENTITY_ENABLE= 2026-07-29T08:48:10.731993903Z + [[ true == \t\r\u\e ]] 2026-07-29T08:48:10.732005331Z + NETWORK_NODE_IDENTITY_ENABLE=' 2026-07-29T08:48:10.732005331Z --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig 2026-07-29T08:48:10.732005331Z --cert-dir=/etc/ovn/ovnkube-node-certs 2026-07-29T08:48:10.732005331Z --cert-duration=24h 2026-07-29T08:48:10.732005331Z ' 2026-07-29T08:48:10.732017111Z + ovn_v4_join_subnet_opt= 2026-07-29T08:48:10.732027350Z + [[ '' != '' ]] 2026-07-29T08:48:10.732027350Z + ovn_v6_join_subnet_opt= 2026-07-29T08:48:10.732039020Z + [[ '' != '' ]] 2026-07-29T08:48:10.732039020Z + ovn_v4_masquerade_subnet_opt= 2026-07-29T08:48:10.732061626Z + [[ 169.254.0.0/17 != '' ]] 2026-07-29T08:48:10.732072257Z + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' 2026-07-29T08:48:10.732072257Z + ovn_v6_masquerade_subnet_opt= 2026-07-29T08:48:10.732081539Z + [[ fd69::/112 != '' ]] 2026-07-29T08:48:10.732090667Z + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' 2026-07-29T08:48:10.732090667Z + ovn_v4_transit_switch_subnet_opt= 2026-07-29T08:48:10.732100559Z + [[ '' != '' ]] 2026-07-29T08:48:10.732100559Z + ovn_v6_transit_switch_subnet_opt= 2026-07-29T08:48:10.732116647Z + [[ '' != '' ]] 2026-07-29T08:48:10.732116647Z + enable_interconnect_flag= 2026-07-29T08:48:10.732365383Z + /usr/bin/ovnkube --help 2026-07-29T08:48:10.732497031Z + grep -q -- --enable-interconnect 2026-07-29T08:48:10.766257188Z + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-133-190.ec2.internal --init-node ip-10-0-133-190.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-133-190.ec2.internal --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true 2026-07-29T08:48:10.811637413Z I0729 08:48:10.811555 3399 config.go:2764] Parsed config file /run/ovnkube-config/ovnkube.conf 2026-07-29T08:48:10.811715237Z I0729 08:48:10.811619 3399 config.go:2765] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.132.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AllowICMPNetworkPolicy:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://a7dfe86e5560c47c48afc61d47eff53d-79bb40d159c06c11.elb.us-east-1.amazonaws.com:6443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.31.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: DPUHostGatewayRepresentorInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName: DPUNodeLeaseRenewInterval:10 DPUNodeLeaseDuration:40 SimulateDPU:false} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology: FRRNamespace:}} 2026-07-29T08:48:10.813099255Z I0729 08:48:10.813063 3399 kube.go:440] Waiting for certificate 2026-07-29T08:48:10.813128537Z I0729 08:48:10.813098 3399 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" 2026-07-29T08:48:10.813173997Z I0729 08:48:10.813135 3399 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" 2026-07-29T08:48:10.813963512Z I0729 08:48:10.813938 3399 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" 2026-07-29T08:48:10.835206275Z I0729 08:48:10.835169 3399 reflector.go:367] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" 2026-07-29T08:48:10.835206275Z I0729 08:48:10.835191 3399 reflector.go:411] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" 2026-07-29T08:48:10.840213667Z I0729 08:48:10.840183 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" logger="kubernetes.io/kube-apiserver-client" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" totalItems=2 duration="4.941238ms" 2026-07-29T08:48:10.840234897Z I0729 08:48:10.840217 3399 reflector.go:446] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" 2026-07-29T08:48:10.840280899Z I0729 08:48:10.840267 3399 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rfqwc" 2026-07-29T08:48:10.846654565Z I0729 08:48:10.846345 3399 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rfqwc" 2026-07-29T08:48:10.846823883Z I0729 08:48:10.846795 3399 reflector.go:373] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" 2026-07-29T08:48:11.813378711Z I0729 08:48:11.813336 3399 kube.go:447] Certificate found 2026-07-29T08:48:11.813870517Z I0729 08:48:11.813846 3399 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" 2026-07-29T08:48:11.815222028Z I0729 08:48:11.815207 3399 server.go:238] Metric Server starts to listen on 127.0.0.1:29103 2026-07-29T08:48:11.861755657Z I0729 08:48:11.861711 3399 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-07-30 08:43:10 +0000 UTC" deadline="2026-07-30 03:17:44.630311844 +0000 UTC" 2026-07-29T08:48:11.861755657Z I0729 08:48:11.861741 3399 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h29m32.768579911s" 2026-07-29T08:48:11.876155133Z I0729 08:48:11.876115 3399 metrics.go:160] OVN Kube log file not specified in config, therefore not starting the log file metric monitor 2026-07-29T08:48:11.887949705Z I0729 08:48:11.887877 3399 node_controller_manager.go:349] Starting the node network controller manager, Mode: full 2026-07-29T08:48:11.888086886Z I0729 08:48:11.888058 3399 factory.go:593] Starting watch factory 2026-07-29T08:48:11.888131098Z I0729 08:48:11.888100 3399 metrics.go:464] Create OVN Metrics Server on address: 127.0.0.1:29105 2026-07-29T08:48:11.888185428Z I0729 08:48:11.888162 3399 server.go:122] MetricServer registers OVS metrics 2026-07-29T08:48:11.888276738Z I0729 08:48:11.888256 3399 reflector.go:367] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888319040Z I0729 08:48:11.888308 3399 reflector.go:411] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888377010Z I0729 08:48:11.888354 3399 reflector.go:367] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888414596Z I0729 08:48:11.888381 3399 reflector.go:411] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888611378Z I0729 08:48:11.888575 3399 reflector.go:367] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888644035Z I0729 08:48:11.888613 3399 reflector.go:411] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888653089Z I0729 08:48:11.888576 3399 reflector.go:367] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.888732626Z I0729 08:48:11.888707 3399 reflector.go:411] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.889088349Z I0729 08:48:11.888724 3399 server.go:126] MetricServer registers OVN DB metrics 2026-07-29T08:48:11.889296185Z I0729 08:48:11.888159 3399 reflector.go:367] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.889355736Z I0729 08:48:11.889333 3399 reflector.go:411] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.890787469Z I0729 08:48:11.888195 3399 reflector.go:367] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.890820315Z I0729 08:48:11.890785 3399 reflector.go:411] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.893589862Z I0729 08:48:11.893559 3399 controller_manager.go:392] Starting the ovnkube controller 2026-07-29T08:48:11.893589862Z I0729 08:48:11.893578 3399 controller_manager.go:397] Waiting up to 5m0s for NBDB zone to match: ip-10-0-133-190.ec2.internal 2026-07-29T08:48:11.893666795Z I0729 08:48:11.893640 3399 controller_manager.go:417] NBDB zone sync took: 47.854µs 2026-07-29T08:48:11.893677123Z I0729 08:48:11.893665 3399 factory.go:593] Starting watch factory 2026-07-29T08:48:11.894775183Z I0729 08:48:11.894744 3399 ovn_db.go:333] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory 2026-07-29T08:48:11.894792792Z I0729 08:48:11.894778 3399 ovn_db.go:330] ovnnb_db.sock found at /var/run/ovn/ 2026-07-29T08:48:11.907556005Z I0729 08:48:11.907518 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=4 duration="18.14031ms" 2026-07-29T08:48:11.907583209Z I0729 08:48:11.907557 3399 reflector.go:446] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.909070754Z I0729 08:48:11.909044 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=62 duration="20.605551ms" 2026-07-29T08:48:11.909151034Z I0729 08:48:11.909137 3399 reflector.go:446] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.911624644Z I0729 08:48:11.911598 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=26 duration="23.240961ms" 2026-07-29T08:48:11.911677963Z I0729 08:48:11.911658 3399 reflector.go:446] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.914575891Z I0729 08:48:11.914549 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=34 duration="25.741437ms" 2026-07-29T08:48:11.914696138Z I0729 08:48:11.914683 3399 reflector.go:446] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.915183922Z I0729 08:48:11.915161 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=46 duration="24.351061ms" 2026-07-29T08:48:11.915351705Z I0729 08:48:11.915338 3399 reflector.go:446] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.915847734Z I0729 08:48:11.915829 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=44 duration="27.149392ms" 2026-07-29T08:48:11.916024198Z I0729 08:48:11.916008 3399 reflector.go:446] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:161" 2026-07-29T08:48:11.923456801Z I0729 08:48:11.923407 3399 ovn_db.go:391] Found OVN NB DB: &{0x1de2f60 /etc/ovn/ovnnb_db.db OVN_Northbound 0} 2026-07-29T08:48:11.923509635Z I0729 08:48:11.923495 3399 ovn_db.go:398] Found OVN SB DB: &{0x1de3220 /etc/ovn/ovnsb_db.db OVN_Southbound 0} 2026-07-29T08:48:11.930317397Z I0729 08:48:11.930286 3399 ovn_db.go:413] Found db is standalone, don't register db_cluster metrics 2026-07-29T08:48:11.930403630Z I0729 08:48:11.930387 3399 server.go:130] MetricServer registers OVN Controller metrics 2026-07-29T08:48:11.935790354Z I0729 08:48:11.935763 3399 server.go:134] MetricServer registers OVN Northd metrics 2026-07-29T08:48:11.939468764Z I0729 08:48:11.939439 3399 metrics.go:471] OVN Metrics Server starts to run ... 2026-07-29T08:48:11.939468764Z I0729 08:48:11.939465 3399 server.go:238] Metric Server starts to listen on 127.0.0.1:29105 2026-07-29T08:48:11.994207953Z I0729 08:48:11.994163 3399 factory.go:2007] *v1.Node informer cache synced successfully 2026-07-29T08:48:11.994207953Z I0729 08:48:11.994191 3399 factory.go:2007] *v1.Service informer cache synced successfully 2026-07-29T08:48:11.994207953Z I0729 08:48:11.994196 3399 factory.go:2007] *v1.EndpointSlice informer cache synced successfully 2026-07-29T08:48:11.994207953Z I0729 08:48:11.994199 3399 factory.go:2007] *v1.Pod informer cache synced successfully 2026-07-29T08:48:11.994207953Z I0729 08:48:11.994202 3399 factory.go:2007] *v1.NetworkPolicy informer cache synced successfully 2026-07-29T08:48:11.994240778Z I0729 08:48:11.994205 3399 factory.go:2007] *v1.Namespace informer cache synced successfully 2026-07-29T08:48:11.994336667Z I0729 08:48:11.994305 3399 reflector.go:367] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" 2026-07-29T08:48:11.994407479Z I0729 08:48:11.994376 3399 reflector.go:411] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" 2026-07-29T08:48:11.994451554Z I0729 08:48:11.994306 3399 reflector.go:367] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" 2026-07-29T08:48:11.994478660Z I0729 08:48:11.994460 3399 reflector.go:411] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" 2026-07-29T08:48:12.000552889Z I0729 08:48:12.000534 3399 factory.go:2007] *v1.EndpointSlice informer cache synced successfully 2026-07-29T08:48:12.000552889Z I0729 08:48:12.000545 3399 factory.go:2007] *v1.Pod informer cache synced successfully 2026-07-29T08:48:12.000552889Z I0729 08:48:12.000549 3399 factory.go:2007] *v1.NetworkPolicy informer cache synced successfully 2026-07-29T08:48:12.000574361Z I0729 08:48:12.000552 3399 factory.go:2007] *v1.Namespace informer cache synced successfully 2026-07-29T08:48:12.000574361Z I0729 08:48:12.000556 3399 factory.go:2007] *v1.Node informer cache synced successfully 2026-07-29T08:48:12.000574361Z I0729 08:48:12.000559 3399 factory.go:2007] *v1.Service informer cache synced successfully 2026-07-29T08:48:12.003835037Z I0729 08:48:12.003806 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" totalItems=1 duration="9.393729ms" 2026-07-29T08:48:12.003835037Z I0729 08:48:12.003830 3399 reflector.go:446] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" 2026-07-29T08:48:12.003955413Z I0729 08:48:12.003939 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" totalItems=1 duration="9.446506ms" 2026-07-29T08:48:12.003977056Z I0729 08:48:12.003958 3399 reflector.go:446] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" 2026-07-29T08:48:12.005939285Z I0729 08:48:12.005918 3399 factory.go:2007] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully 2026-07-29T08:48:12.005939285Z I0729 08:48:12.005930 3399 factory.go:2007] *v1alpha1.AdminNetworkPolicy informer cache synced successfully 2026-07-29T08:48:12.006011961Z I0729 08:48:12.005997 3399 reflector.go:367] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.006021412Z I0729 08:48:12.006014 3399 reflector.go:411] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.009334905Z I0729 08:48:12.009316 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="3.277892ms" 2026-07-29T08:48:12.009350852Z I0729 08:48:12.009335 3399 reflector.go:446] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.011346714Z I0729 08:48:12.011332 3399 factory.go:2007] *v1.EgressIP informer cache synced successfully 2026-07-29T08:48:12.011406628Z I0729 08:48:12.011390 3399 reflector.go:367] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.011417357Z I0729 08:48:12.011406 3399 reflector.go:411] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.015492363Z I0729 08:48:12.015468 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="4.036922ms" 2026-07-29T08:48:12.015508200Z I0729 08:48:12.015490 3399 reflector.go:446] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.016417622Z I0729 08:48:12.016400 3399 factory.go:2007] *v1.EgressFirewall informer cache synced successfully 2026-07-29T08:48:12.016549251Z I0729 08:48:12.016532 3399 reflector.go:367] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.016558518Z I0729 08:48:12.016550 3399 reflector.go:411] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.019632101Z I0729 08:48:12.019603 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="3.032065ms" 2026-07-29T08:48:12.019632101Z I0729 08:48:12.019625 3399 reflector.go:446] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.021601744Z I0729 08:48:12.021583 3399 factory.go:2007] *v1.EgressQoS informer cache synced successfully 2026-07-29T08:48:12.021653582Z I0729 08:48:12.021638 3399 reflector.go:367] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.021653582Z I0729 08:48:12.021649 3399 reflector.go:411] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.025873019Z I0729 08:48:12.025848 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="4.180273ms" 2026-07-29T08:48:12.025873019Z I0729 08:48:12.025866 3399 reflector.go:446] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.026840342Z I0729 08:48:12.026823 3399 factory.go:2007] *v1.EgressService informer cache synced successfully 2026-07-29T08:48:12.026922503Z I0729 08:48:12.026905 3399 reflector.go:367] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.026931743Z I0729 08:48:12.026920 3399 reflector.go:411] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.029819834Z I0729 08:48:12.029802 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="2.859895ms" 2026-07-29T08:48:12.029829550Z I0729 08:48:12.029821 3399 reflector.go:446] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.032804640Z I0729 08:48:12.032788 3399 factory.go:2007] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully 2026-07-29T08:48:12.032978252Z I0729 08:48:12.032959 3399 reflector.go:367] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" 2026-07-29T08:48:12.032987932Z I0729 08:48:12.032978 3399 reflector.go:411] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" 2026-07-29T08:48:12.036301975Z I0729 08:48:12.036279 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" totalItems=2 duration="3.278786ms" 2026-07-29T08:48:12.036339253Z I0729 08:48:12.036306 3399 reflector.go:446] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" 2026-07-29T08:48:12.038259838Z I0729 08:48:12.038244 3399 factory.go:2007] *v1.NetworkAttachmentDefinition informer cache synced successfully 2026-07-29T08:48:12.038331572Z I0729 08:48:12.038300 3399 reflector.go:367] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.038342368Z I0729 08:48:12.038332 3399 reflector.go:411] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.038410102Z I0729 08:48:12.038389 3399 reflector.go:367] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.038446742Z I0729 08:48:12.038435 3399 reflector.go:411] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.041370621Z I0729 08:48:12.041348 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="2.870185ms" 2026-07-29T08:48:12.041390217Z I0729 08:48:12.041367 3399 reflector.go:446] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.041456754Z I0729 08:48:12.041437 3399 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" totalItems=1 duration="3.08319ms" 2026-07-29T08:48:12.041465411Z I0729 08:48:12.041455 3399 reflector.go:446] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" 2026-07-29T08:48:12.043330168Z I0729 08:48:12.043309 3399 factory.go:2007] *v1.UserDefinedNetwork informer cache synced successfully 2026-07-29T08:48:12.043330168Z I0729 08:48:12.043322 3399 factory.go:2007] *v1.ClusterUserDefinedNetwork informer cache synced successfully 2026-07-29T08:48:12.043339893Z I0729 08:48:12.043329 3399 factory.go:712] Watch Factory start up complete, took: 149.662964ms 2026-07-29T08:48:12.043339893Z I0729 08:48:12.043336 3399 controller_manager.go:428] Waiting up to 5m0s for a node to have "ip-10-0-133-190.ec2.internal" zone 2026-07-29T08:48:12.099596059Z I0729 08:48:12.099560 3399 factory.go:2007] *v1alpha1.AdminNetworkPolicy informer cache synced successfully 2026-07-29T08:48:12.099596059Z I0729 08:48:12.099577 3399 factory.go:2007] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully 2026-07-29T08:48:12.104753916Z I0729 08:48:12.104736 3399 factory.go:2007] *v1.EgressIP informer cache synced successfully 2026-07-29T08:48:12.109928132Z I0729 08:48:12.109909 3399 factory.go:2007] *v1.EgressFirewall informer cache synced successfully 2026-07-29T08:48:12.115090412Z I0729 08:48:12.115072 3399 factory.go:2007] *v1.EgressQoS informer cache synced successfully 2026-07-29T08:48:12.120245931Z I0729 08:48:12.120226 3399 factory.go:2007] *v1.EgressService informer cache synced successfully 2026-07-29T08:48:12.125406492Z I0729 08:48:12.125387 3399 factory.go:2007] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully 2026-07-29T08:48:12.130562723Z I0729 08:48:12.130544 3399 factory.go:2007] *v1.NetworkAttachmentDefinition informer cache synced successfully 2026-07-29T08:48:12.135726178Z I0729 08:48:12.135706 3399 factory.go:2007] *v1.UserDefinedNetwork informer cache synced successfully 2026-07-29T08:48:12.135726178Z I0729 08:48:12.135717 3399 factory.go:2007] *v1.ClusterUserDefinedNetwork informer cache synced successfully 2026-07-29T08:48:12.135738242Z I0729 08:48:12.135725 3399 factory.go:712] Watch Factory start up complete, took: 247.67267ms 2026-07-29T08:48:12.135821225Z I0729 08:48:12.135805 3399 default_node_network_controller.go:193] Enable node proxy healthz server on 0.0.0.0:10256 2026-07-29T08:48:12.182478362Z I0729 08:48:12.182441 3399 default_node_network_controller.go:737] Initializing the default node network controller 2026-07-29T08:48:12.257368534Z I0729 08:48:12.257336 3399 udn_isolation.go:101] Starting UDN host isolation manager 2026-07-29T08:48:12.264111072Z I0729 08:48:12.264089 3399 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service 2026-07-29T08:48:12.326046797Z I0729 08:48:12.326011 3399 controller.go:153] Adding controller udn-host-isolation-manager event handlers 2026-07-29T08:48:12.326121942Z I0729 08:48:12.326104 3399 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" 2026-07-29T08:48:12.326130414Z I0729 08:48:12.326122 3399 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" 2026-07-29T08:48:12.359725225Z I0729 08:48:12.359685 3399 controller.go:177] Starting controller udn-host-isolation-manager with 1 workers 2026-07-29T08:48:12.359769572Z I0729 08:48:12.359758 3399 default_node_network_controller.go:852] Node ip-10-0-133-190.ec2.internal ready for ovn initialization with subnet 10.133.0.0/23 2026-07-29T08:48:12.365424706Z I0729 08:48:12.365401 3399 kube.go:266] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.133.190"] k8s.ovn.org/zone-name:ip-10-0-133-190.ec2.internal] on node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.382615048Z I0729 08:48:12.382578 3399 gateway_init.go:241] Initializing Gateway Functionality for Gateway PreStart 2026-07-29T08:48:12.388701964Z I0729 08:48:12.388674 3399 helper_linux.go:96] Provided gateway interface "br-ex", found as index: 5 2026-07-29T08:48:12.388818962Z I0729 08:48:12.388803 3399 helper_linux.go:121] Found default gateway interface br-ex 10.0.128.1 2026-07-29T08:48:12.388975633Z I0729 08:48:12.388959 3399 gateway_init.go:270] Preparing Gateway 2026-07-29T08:48:12.388993896Z I0729 08:48:12.388973 3399 gateway_shared_intf.go:1741] Creating new gateway 2026-07-29T08:48:12.460280983Z I0729 08:48:12.460243 3399 iptables.go:111] Creating table: filter chain: FORWARD 2026-07-29T08:48:12.466446704Z I0729 08:48:12.466413 3399 iptables.go:111] Creating table: filter chain: OUTPUT 2026-07-29T08:48:12.472663963Z I0729 08:48:12.472638 3399 gateway_shared_intf.go:1869] Gateway Creation Complete 2026-07-29T08:48:12.472690781Z I0729 08:48:12.472655 3399 kube.go:266] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-133-190.ec2.internal","mac-address":"02:98:b3:77:59:bf","ip-addresses":["10.0.133.190/20"],"ip-address":"10.0.133.190/20","next-hops":["10.0.128.1"],"next-hop":"10.0.128.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3b78e20e-60b0-44ab-80ad-255ec40ceb5f k8s.ovn.org/node-encap-ips:["10.0.133.190"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.133.190/20"} k8s.ovn.org/zone-name:ip-10-0-133-190.ec2.internal] on node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.494001016Z I0729 08:48:12.493966 3399 default_node_network_controller.go:549] Node connection status = connected 2026-07-29T08:48:12.543510884Z I0729 08:48:12.543461 3399 controller_manager.go:451] Waiting for node in zone sync took: 500.108713ms 2026-07-29T08:48:12.547263303Z I0729 08:48:12.547227 3399 controller_manager.go:644] Switching to transit router for layer2 networks 2026-07-29T08:48:12.547289894Z I0729 08:48:12.547244 3399 kube.go:266] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.563415714Z I0729 08:48:12.563375 3399 metrics.go:160] OVN Kube log file not specified in config, therefore not starting the log file metric monitor 2026-07-29T08:48:12.563965469Z I0729 08:48:12.563944 3399 addressset_manager.go:173] Starting pod-selector-address-set-manager controller 2026-07-29T08:48:12.563984654Z I0729 08:48:12.563976 3399 controller.go:153] Adding controller pod-selector-address-set-manager-pod event handlers 2026-07-29T08:48:12.564033875Z I0729 08:48:12.564018 3399 controller.go:153] Adding controller pod-selector-address-set-manager-node event handlers 2026-07-29T08:48:12.564044344Z I0729 08:48:12.564029 3399 controller.go:153] Adding controller pod-selector-address-set-manager-namespace event handlers 2026-07-29T08:48:12.564073642Z I0729 08:48:12.564050 3399 shared_informer.go:349] "Waiting for caches to sync" controller="pod-selector-address-set-manager-pod" 2026-07-29T08:48:12.564083227Z I0729 08:48:12.564074 3399 shared_informer.go:356] "Caches are synced" controller="pod-selector-address-set-manager-pod" 2026-07-29T08:48:12.564092415Z I0729 08:48:12.564078 3399 shared_informer.go:349] "Waiting for caches to sync" controller="pod-selector-address-set-manager-node" 2026-07-29T08:48:12.564101354Z I0729 08:48:12.564095 3399 shared_informer.go:356] "Caches are synced" controller="pod-selector-address-set-manager-node" 2026-07-29T08:48:12.564161995Z I0729 08:48:12.564142 3399 shared_informer.go:349] "Waiting for caches to sync" controller="pod-selector-address-set-manager-namespace" 2026-07-29T08:48:12.564172458Z I0729 08:48:12.564161 3399 shared_informer.go:356] "Caches are synced" controller="pod-selector-address-set-manager-namespace" 2026-07-29T08:48:12.564244498Z I0729 08:48:12.564231 3399 controller.go:177] Starting controller pod-selector-address-set-manager-pod with 1 workers 2026-07-29T08:48:12.564255243Z I0729 08:48:12.564247 3399 controller.go:177] Starting controller pod-selector-address-set-manager-namespace with 1 workers 2026-07-29T08:48:12.564264583Z I0729 08:48:12.564255 3399 controller.go:177] Starting controller pod-selector-address-set-manager-node with 1 workers 2026-07-29T08:48:12.564287437Z I0729 08:48:12.564264 3399 controller.go:177] Starting controller pod-selector-address-set-manager-addrset with 1 workers 2026-07-29T08:48:12.564488184Z I0729 08:48:12.564472 3399 services_controller.go:148] Creating services controller for network=default 2026-07-29T08:48:12.564601752Z I0729 08:48:12.564576 3399 udn_enabled_svc.go:84] Waiting for service informer to sync 2026-07-29T08:48:12.564718983Z I0729 08:48:12.564706 3399 controller.go:153] Adding controller node-topology-node event handlers 2026-07-29T08:48:12.564742306Z I0729 08:48:12.564731 3399 shared_informer.go:349] "Waiting for caches to sync" controller="node-topology-node" 2026-07-29T08:48:12.564742306Z I0729 08:48:12.564740 3399 shared_informer.go:356] "Caches are synced" controller="node-topology-node" 2026-07-29T08:48:12.564753170Z I0729 08:48:12.564746 3399 controller.go:177] Starting controller node-topology-node with 15 workers 2026-07-29T08:48:12.564851607Z I0729 08:48:12.564840 3399 controller.go:153] Adding controller [zone-nad-controller NAD controller] event handlers 2026-07-29T08:48:12.564907981Z I0729 08:48:12.564884 3399 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" 2026-07-29T08:48:12.564920859Z I0729 08:48:12.564912 3399 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" 2026-07-29T08:48:12.564929612Z I0729 08:48:12.564923 3399 nad_controller.go:971] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes 2026-07-29T08:48:12.565195569Z I0729 08:48:12.565173 3399 udn_enabled_svc.go:99] Performing full resync 2026-07-29T08:48:12.565195569Z I0729 08:48:12.565186 3399 controller.go:177] Starting controller [zone-nad-controller NAD controller] with 1 workers 2026-07-29T08:48:12.565260311Z I0729 08:48:12.565245 3399 network_controller.go:376] [zone-nad-controller network controller]: syncing all networks 2026-07-29T08:48:12.565299702Z I0729 08:48:12.565277 3399 nad_controller.go:1010] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 39.107µs 2026-07-29T08:48:12.565321738Z I0729 08:48:12.565308 3399 network_controller.go:401] [zone-nad-controller network controller]: finished syncing network default, took 49.936µs 2026-07-29T08:48:12.565321738Z I0729 08:48:12.565316 3399 network_controller.go:383] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 72.488µs 2026-07-29T08:48:12.565329530Z I0729 08:48:12.565321 3399 controller.go:177] Starting controller [zone-nad-controller network controller] with 1 workers 2026-07-29T08:48:12.565336154Z I0729 08:48:12.565328 3399 nad_controller.go:552] [zone-nad-controller NAD controller]: started 2026-07-29T08:48:12.565342612Z I0729 08:48:12.565336 3399 default_network_controller.go:331] Starting the default network controller 2026-07-29T08:48:12.565428391Z I0729 08:48:12.565414 3399 network_controller.go:401] [zone-nad-controller network controller]: finished syncing network default, took 13.746µs 2026-07-29T08:48:12.565536432Z I0729 08:48:12.565522 3399 udn_enabled_svc.go:103] Waiting for handler to sync 2026-07-29T08:48:12.565547984Z I0729 08:48:12.565535 3399 udn_enabled_svc.go:108] Starting worker 2026-07-29T08:48:12.575321727Z I0729 08:48:12.575299 3399 default_network_controller.go:637] Cleaning External Gateway ECMP routes 2026-07-29T08:48:12.575386569Z I0729 08:48:12.575372 3399 repair.go:36] Syncing exgw routes took 51.975µs 2026-07-29T08:48:12.575402921Z I0729 08:48:12.575393 3399 default_network_controller.go:656] Starting all the Watchers... 2026-07-29T08:48:12.575570669Z I0729 08:48:12.575550 3399 namespace.go:89] [openshift-dns] adding namespace 2026-07-29T08:48:12.575570669Z I0729 08:48:12.575554 3399 namespace.go:89] [openshift-cloud-credential-operator] adding namespace 2026-07-29T08:48:12.575570669Z I0729 08:48:12.575558 3399 namespace.go:89] [openshift-kube-scheduler] adding namespace 2026-07-29T08:48:12.575570669Z I0729 08:48:12.575560 3399 namespace.go:89] [openshift-machine-api] adding namespace 2026-07-29T08:48:12.575590967Z I0729 08:48:12.575573 3399 namespace.go:89] [open-cluster-management-agent-addon] adding namespace 2026-07-29T08:48:12.575590967Z I0729 08:48:12.575578 3399 namespace.go:89] [openshift-host-network] adding namespace 2026-07-29T08:48:12.575590967Z I0729 08:48:12.575578 3399 namespace.go:89] [openshift-image-registry] adding namespace 2026-07-29T08:48:12.575590967Z I0729 08:48:12.575588 3399 namespace.go:89] [default] adding namespace 2026-07-29T08:48:12.575600547Z I0729 08:48:12.575592 3399 namespace.go:89] [openshift-authentication] adding namespace 2026-07-29T08:48:12.575600547Z I0729 08:48:12.575594 3399 namespace.go:89] [openshift-ingress] adding namespace 2026-07-29T08:48:12.575609357Z I0729 08:48:12.575596 3399 namespace.go:89] [openshift-monitoring] adding namespace 2026-07-29T08:48:12.575609357Z I0729 08:48:12.575594 3399 namespace.go:89] [openshift-network-node-identity] adding namespace 2026-07-29T08:48:12.575609357Z I0729 08:48:12.575595 3399 namespace.go:89] [openshift-ingress-operator] adding namespace 2026-07-29T08:48:12.575633890Z I0729 08:48:12.575619 3399 namespace.go:89] [openshift-operator-lifecycle-manager] adding namespace 2026-07-29T08:48:12.575684198Z I0729 08:48:12.575552 3399 namespace.go:89] [openshift-kube-scheduler-operator] adding namespace 2026-07-29T08:48:12.576050578Z I0729 08:48:12.576034 3399 namespace.go:93] [openshift-dns] adding namespace took 470.223µs 2026-07-29T08:48:12.576063141Z I0729 08:48:12.576049 3399 namespace.go:89] [openshift-kube-apiserver-operator] adding namespace 2026-07-29T08:48:12.576351114Z I0729 08:48:12.576328 3399 namespace.go:93] [openshift-kube-apiserver-operator] adding namespace took 274.017µs 2026-07-29T08:48:12.576351114Z I0729 08:48:12.576347 3399 namespace.go:89] [openshift-network-operator] adding namespace 2026-07-29T08:48:12.576631211Z I0729 08:48:12.576608 3399 namespace.go:93] [openshift-network-operator] adding namespace took 254.583µs 2026-07-29T08:48:12.576631211Z I0729 08:48:12.576623 3399 namespace.go:89] [openshift-marketplace] adding namespace 2026-07-29T08:48:12.576993845Z I0729 08:48:12.576975 3399 namespace.go:93] [openshift-marketplace] adding namespace took 346.661µs 2026-07-29T08:48:12.577272667Z I0729 08:48:12.577256 3399 namespace.go:93] [openshift-cloud-credential-operator] adding namespace took 1.690021ms 2026-07-29T08:48:12.577272667Z I0729 08:48:12.577269 3399 namespace.go:89] [openshift-apiserver-operator] adding namespace 2026-07-29T08:48:12.577550485Z I0729 08:48:12.577533 3399 namespace.go:93] [openshift-kube-scheduler] adding namespace took 1.964328ms 2026-07-29T08:48:12.577550485Z I0729 08:48:12.577542 3399 namespace.go:89] [openshift-cluster-samples-operator] adding namespace 2026-07-29T08:48:12.577816022Z I0729 08:48:12.577796 3399 namespace.go:93] [openshift-machine-api] adding namespace took 2.220902ms 2026-07-29T08:48:12.577816022Z I0729 08:48:12.577812 3399 namespace.go:89] [openshift-network-console] adding namespace 2026-07-29T08:48:12.578090257Z I0729 08:48:12.578072 3399 namespace.go:93] [open-cluster-management-agent-addon] adding namespace took 2.489354ms 2026-07-29T08:48:12.578106336Z I0729 08:48:12.578090 3399 namespace.go:89] [openshift-cluster-version] adding namespace 2026-07-29T08:48:12.578853747Z I0729 08:48:12.578835 3399 namespace.go:93] [openshift-host-network] adding namespace took 3.248549ms 2026-07-29T08:48:12.578853747Z I0729 08:48:12.578849 3399 namespace.go:89] [openshift-cloud-controller-manager] adding namespace 2026-07-29T08:48:12.579146389Z I0729 08:48:12.579114 3399 namespace.go:93] [openshift-image-registry] adding namespace took 3.525248ms 2026-07-29T08:48:12.579187698Z I0729 08:48:12.579174 3399 namespace.go:89] [openshift] adding namespace 2026-07-29T08:48:12.579406990Z I0729 08:48:12.579391 3399 namespace.go:93] [default] adding namespace took 3.798817ms 2026-07-29T08:48:12.579406990Z I0729 08:48:12.579402 3399 namespace.go:89] [openshift-dns-operator] adding namespace 2026-07-29T08:48:12.579701248Z I0729 08:48:12.579682 3399 namespace.go:93] [openshift-authentication] adding namespace took 4.082832ms 2026-07-29T08:48:12.579701248Z I0729 08:48:12.579696 3399 namespace.go:89] [kube-node-lease] adding namespace 2026-07-29T08:48:12.580033976Z I0729 08:48:12.580015 3399 namespace.go:93] [openshift-ingress] adding namespace took 4.412055ms 2026-07-29T08:48:12.580052376Z I0729 08:48:12.580036 3399 namespace.go:89] [kube-system] adding namespace 2026-07-29T08:48:12.580291699Z I0729 08:48:12.580277 3399 namespace.go:93] [openshift-monitoring] adding namespace took 4.672141ms 2026-07-29T08:48:12.580303959Z I0729 08:48:12.580291 3399 namespace.go:89] [openshift-controller-manager] adding namespace 2026-07-29T08:48:12.580532376Z I0729 08:48:12.580517 3399 namespace.go:93] [openshift-network-node-identity] adding namespace took 4.909593ms 2026-07-29T08:48:12.580542646Z I0729 08:48:12.580532 3399 namespace.go:89] [openshift-console-operator] adding namespace 2026-07-29T08:48:12.580768263Z I0729 08:48:12.580748 3399 namespace.go:93] [openshift-ingress-operator] adding namespace took 5.138639ms 2026-07-29T08:48:12.580768263Z I0729 08:48:12.580762 3399 namespace.go:89] [openshift-apiserver] adding namespace 2026-07-29T08:48:12.581071990Z I0729 08:48:12.581047 3399 namespace.go:93] [openshift-operator-lifecycle-manager] adding namespace took 5.417903ms 2026-07-29T08:48:12.581071990Z I0729 08:48:12.581067 3399 namespace.go:89] [openshift-network-diagnostics] adding namespace 2026-07-29T08:48:12.581434696Z I0729 08:48:12.581418 3399 namespace.go:93] [openshift-kube-scheduler-operator] adding namespace took 5.741284ms 2026-07-29T08:48:12.581459406Z I0729 08:48:12.581435 3399 namespace.go:89] [openshift-ovn-kubernetes] adding namespace 2026-07-29T08:48:12.581712087Z I0729 08:48:12.581693 3399 namespace.go:93] [openshift-apiserver-operator] adding namespace took 4.416001ms 2026-07-29T08:48:12.581723228Z I0729 08:48:12.581713 3399 namespace.go:89] [openshift-route-controller-manager] adding namespace 2026-07-29T08:48:12.582042843Z I0729 08:48:12.582026 3399 namespace.go:93] [openshift-cluster-samples-operator] adding namespace took 4.47777ms 2026-07-29T08:48:12.582054997Z I0729 08:48:12.582044 3399 namespace.go:89] [openshift-config] adding namespace 2026-07-29T08:48:12.582306067Z I0729 08:48:12.582293 3399 namespace.go:93] [openshift-network-console] adding namespace took 4.476163ms 2026-07-29T08:48:12.582314703Z I0729 08:48:12.582304 3399 namespace.go:89] [openshift-infra] adding namespace 2026-07-29T08:48:12.582528839Z I0729 08:48:12.582515 3399 namespace.go:93] [openshift-cluster-version] adding namespace took 4.419089ms 2026-07-29T08:48:12.582555070Z I0729 08:48:12.582545 3399 namespace.go:89] [openshift-operators] adding namespace 2026-07-29T08:48:12.582778579Z I0729 08:48:12.582764 3399 namespace.go:93] [openshift-cloud-controller-manager] adding namespace took 3.908571ms 2026-07-29T08:48:12.582794792Z I0729 08:48:12.582778 3399 namespace.go:89] [openshift-kube-controller-manager] adding namespace 2026-07-29T08:48:12.583010351Z I0729 08:48:12.582995 3399 namespace.go:93] [openshift] adding namespace took 3.811886ms 2026-07-29T08:48:12.583010351Z I0729 08:48:12.583008 3399 namespace.go:89] [openshift-node] adding namespace 2026-07-29T08:48:12.583228805Z I0729 08:48:12.583216 3399 namespace.go:93] [openshift-dns-operator] adding namespace took 3.809435ms 2026-07-29T08:48:12.583237399Z I0729 08:48:12.583228 3399 namespace.go:89] [openshift-cluster-storage-operator] adding namespace 2026-07-29T08:48:12.583430512Z I0729 08:48:12.583418 3399 namespace.go:93] [kube-node-lease] adding namespace took 3.71782ms 2026-07-29T08:48:12.583438839Z I0729 08:48:12.583429 3399 namespace.go:89] [openshift-kube-controller-manager-operator] adding namespace 2026-07-29T08:48:12.583673478Z I0729 08:48:12.583661 3399 namespace.go:93] [kube-system] adding namespace took 3.620458ms 2026-07-29T08:48:12.583682024Z I0729 08:48:12.583671 3399 namespace.go:89] [openshift-multus] adding namespace 2026-07-29T08:48:12.583862451Z I0729 08:48:12.583849 3399 namespace.go:93] [openshift-controller-manager] adding namespace took 3.552736ms 2026-07-29T08:48:12.583870352Z I0729 08:48:12.583861 3399 namespace.go:89] [openshift-machine-config-operator] adding namespace 2026-07-29T08:48:12.584112302Z I0729 08:48:12.584097 3399 namespace.go:93] [openshift-console-operator] adding namespace took 3.560701ms 2026-07-29T08:48:12.584123400Z I0729 08:48:12.584111 3399 namespace.go:89] [kube-public] adding namespace 2026-07-29T08:48:12.584352090Z I0729 08:48:12.584341 3399 namespace.go:93] [openshift-apiserver] adding namespace took 3.574034ms 2026-07-29T08:48:12.584359501Z I0729 08:48:12.584351 3399 namespace.go:89] [openshift-cloud-network-config-controller] adding namespace 2026-07-29T08:48:12.584552428Z I0729 08:48:12.584541 3399 namespace.go:93] [openshift-network-diagnostics] adding namespace took 3.468549ms 2026-07-29T08:48:12.584559788Z I0729 08:48:12.584553 3399 namespace.go:89] [openshift-user-workload-monitoring] adding namespace 2026-07-29T08:48:12.584848402Z I0729 08:48:12.584831 3399 namespace.go:93] [openshift-ovn-kubernetes] adding namespace took 3.389511ms 2026-07-29T08:48:12.584859484Z I0729 08:48:12.584847 3399 namespace.go:89] [openshift-cluster-machine-approver] adding namespace 2026-07-29T08:48:12.585164145Z I0729 08:48:12.585148 3399 namespace.go:93] [openshift-route-controller-manager] adding namespace took 3.42861ms 2026-07-29T08:48:12.585177318Z I0729 08:48:12.585165 3399 namespace.go:89] [openshift-authentication-operator] adding namespace 2026-07-29T08:48:12.585394154Z I0729 08:48:12.585381 3399 namespace.go:93] [openshift-config] adding namespace took 3.332592ms 2026-07-29T08:48:12.585402198Z I0729 08:48:12.585393 3399 namespace.go:89] [openshift-console-user-settings] adding namespace 2026-07-29T08:48:12.585643817Z I0729 08:48:12.585630 3399 namespace.go:93] [openshift-infra] adding namespace took 3.319613ms 2026-07-29T08:48:12.585651713Z I0729 08:48:12.585647 3399 namespace.go:89] [openshift-cluster-node-tuning-operator] adding namespace 2026-07-29T08:48:12.585880171Z I0729 08:48:12.585868 3399 namespace.go:93] [openshift-operators] adding namespace took 3.316013ms 2026-07-29T08:48:12.585888788Z I0729 08:48:12.585880 3399 namespace.go:89] [openshift-config-operator] adding namespace 2026-07-29T08:48:12.586161378Z I0729 08:48:12.586145 3399 namespace.go:93] [openshift-kube-controller-manager] adding namespace took 3.360607ms 2026-07-29T08:48:12.586176333Z I0729 08:48:12.586163 3399 namespace.go:89] [openshift-etcd] adding namespace 2026-07-29T08:48:12.586391482Z I0729 08:48:12.586379 3399 namespace.go:93] [openshift-node] adding namespace took 3.366486ms 2026-07-29T08:48:12.586399956Z I0729 08:48:12.586390 3399 namespace.go:89] [openshift-service-ca-operator] adding namespace 2026-07-29T08:48:12.586646889Z I0729 08:48:12.586635 3399 namespace.go:93] [openshift-cluster-storage-operator] adding namespace took 3.401954ms 2026-07-29T08:48:12.586653843Z I0729 08:48:12.586649 3399 namespace.go:89] [openshift-config-managed] adding namespace 2026-07-29T08:48:12.586856218Z I0729 08:48:12.586843 3399 namespace.go:93] [openshift-kube-controller-manager-operator] adding namespace took 3.409331ms 2026-07-29T08:48:12.586856218Z I0729 08:48:12.586853 3399 namespace.go:89] [openshift-kube-apiserver] adding namespace 2026-07-29T08:48:12.587102556Z I0729 08:48:12.587088 3399 namespace.go:93] [openshift-multus] adding namespace took 3.411096ms 2026-07-29T08:48:12.587102556Z I0729 08:48:12.587100 3399 namespace.go:89] [openshift-ingress-canary] adding namespace 2026-07-29T08:48:12.587322834Z I0729 08:48:12.587311 3399 namespace.go:93] [openshift-machine-config-operator] adding namespace took 3.445319ms 2026-07-29T08:48:12.587330023Z I0729 08:48:12.587324 3399 namespace.go:89] [openshift-insights] adding namespace 2026-07-29T08:48:12.587534647Z I0729 08:48:12.587523 3399 namespace.go:93] [kube-public] adding namespace took 3.407852ms 2026-07-29T08:48:12.587544156Z I0729 08:48:12.587536 3399 namespace.go:89] [openshift-cluster-csi-drivers] adding namespace 2026-07-29T08:48:12.587753273Z I0729 08:48:12.587741 3399 namespace.go:93] [openshift-cloud-network-config-controller] adding namespace took 3.385733ms 2026-07-29T08:48:12.587761476Z I0729 08:48:12.587752 3399 namespace.go:89] [openshift-controller-manager-operator] adding namespace 2026-07-29T08:48:12.587987782Z I0729 08:48:12.587974 3399 namespace.go:93] [openshift-user-workload-monitoring] adding namespace took 3.417588ms 2026-07-29T08:48:12.587998796Z I0729 08:48:12.587986 3399 namespace.go:89] [openshift-console] adding namespace 2026-07-29T08:48:12.588248894Z I0729 08:48:12.588235 3399 namespace.go:93] [openshift-cluster-machine-approver] adding namespace took 3.384096ms 2026-07-29T08:48:12.588248894Z I0729 08:48:12.588245 3399 namespace.go:89] [open-cluster-management-c1f2f071-752f-4f7d-a2b1-b6af2b070] adding namespace 2026-07-29T08:48:12.588529954Z I0729 08:48:12.588507 3399 namespace.go:93] [openshift-authentication-operator] adding namespace took 3.33712ms 2026-07-29T08:48:12.588529954Z I0729 08:48:12.588517 3399 namespace.go:89] [openshift-kube-storage-version-migrator-operator] adding namespace 2026-07-29T08:48:12.588778140Z I0729 08:48:12.588762 3399 namespace.go:93] [openshift-console-user-settings] adding namespace took 3.363796ms 2026-07-29T08:48:12.589026418Z I0729 08:48:12.589012 3399 namespace.go:93] [openshift-cluster-node-tuning-operator] adding namespace took 3.361207ms 2026-07-29T08:48:12.589314353Z I0729 08:48:12.589300 3399 namespace.go:93] [openshift-config-operator] adding namespace took 3.413724ms 2026-07-29T08:48:12.589576509Z I0729 08:48:12.589560 3399 namespace.go:93] [openshift-etcd] adding namespace took 3.392008ms 2026-07-29T08:48:12.589829636Z I0729 08:48:12.589807 3399 namespace.go:93] [openshift-service-ca-operator] adding namespace took 3.411691ms 2026-07-29T08:48:12.590118236Z I0729 08:48:12.590101 3399 namespace.go:93] [openshift-config-managed] adding namespace took 3.448477ms 2026-07-29T08:48:12.590368647Z I0729 08:48:12.590350 3399 namespace.go:93] [openshift-kube-apiserver] adding namespace took 3.492048ms 2026-07-29T08:48:12.590615363Z I0729 08:48:12.590595 3399 namespace.go:93] [openshift-ingress-canary] adding namespace took 3.491472ms 2026-07-29T08:48:12.590875490Z I0729 08:48:12.590862 3399 namespace.go:93] [openshift-insights] adding namespace took 3.533194ms 2026-07-29T08:48:12.591122647Z I0729 08:48:12.591102 3399 namespace.go:93] [openshift-cluster-csi-drivers] adding namespace took 3.561455ms 2026-07-29T08:48:12.591363624Z I0729 08:48:12.591348 3399 namespace.go:93] [openshift-controller-manager-operator] adding namespace took 3.59161ms 2026-07-29T08:48:12.591551806Z I0729 08:48:12.591536 3399 namespace.go:93] [openshift-console] adding namespace took 3.54467ms 2026-07-29T08:48:12.591867599Z I0729 08:48:12.591854 3399 namespace.go:93] [open-cluster-management-c1f2f071-752f-4f7d-a2b1-b6af2b070] adding namespace took 3.60292ms 2026-07-29T08:48:12.592057898Z I0729 08:48:12.592044 3399 namespace.go:93] [openshift-kube-storage-version-migrator-operator] adding namespace took 3.522938ms 2026-07-29T08:48:12.592141101Z W0729 08:48:12.592130 3399 master.go:482] Unable to parse remote node ip-10-0-141-66.ec2.internal chassis-id annotation. Chassis may be removed during sync 2026-07-29T08:48:12.592273965Z I0729 08:48:12.592256 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.592320458Z E0729 08:48:12.592308 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.592399310Z I0729 08:48:12.592388 3399 default_network_controller.go:564] Node "ip-10-0-131-230.ec2.internal" in remote zone "ip-10-0-131-230.ec2.internal", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.593406596Z I0729 08:48:12.593383 3399 zone_ic_handler.go:198] Time taken to create transit switch: 367.361µs 2026-07-29T08:48:12.593406596Z I0729 08:48:12.593397 3399 zone_ic_handler.go:280] Creating interconnect resources for remote zone node ip-10-0-131-230.ec2.internal for the network default 2026-07-29T08:48:12.594463044Z I0729 08:48:12.594439 3399 zone_ic_handler.go:285] Creating Interconnect resources for node "ip-10-0-131-230.ec2.internal" on network "default" took: 1.449634ms 2026-07-29T08:48:12.594588264Z I0729 08:48:12.594573 3399 master.go:559] Adding or Updating local node "ip-10-0-133-190.ec2.internal" for network "default" 2026-07-29T08:48:12.603523507Z W0729 08:48:12.603497 3399 gateway.go:219] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-4q6x4: pod openshift-network-diagnostics/network-check-target-4q6x4: no pod IPs found 2026-07-29T08:48:12.603523507Z W0729 08:48:12.603511 3399 gateway.go:219] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-8t5kw: pod kube-system/global-pull-secret-syncer-8t5kw: no pod IPs found 2026-07-29T08:48:12.603553812Z W0729 08:48:12.603533 3399 gateway.go:219] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-js5hz: pod openshift-multus/network-metrics-daemon-js5hz: no pod IPs found 2026-07-29T08:48:12.604593874Z I0729 08:48:12.604570 3399 zone_ic_handler.go:206] Creating interconnect resources for local zone node ip-10-0-133-190.ec2.internal for the network default 2026-07-29T08:48:12.604883327Z I0729 08:48:12.604819 3399 zone_ic_handler.go:198] Time taken to create transit switch: 235.536µs 2026-07-29T08:48:12.605753829Z I0729 08:48:12.605736 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.605799372Z E0729 08:48:12.605787 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.616869633Z I0729 08:48:12.616841 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.616928359Z E0729 08:48:12.616915 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.620754647Z + [[ 1 -lt 90 ]] 2026-07-29T08:48:12.620770004Z + add_garp_drop_flow br-ex 2026-07-29T08:48:12.620797689Z + local bridge=br-ex 2026-07-29T08:48:12.620806784Z + local cookie=0x0305 2026-07-29T08:48:12.620813292Z + local priority=499 2026-07-29T08:48:12.621160437Z ++ ovs-vsctl list-ports br-ex 2026-07-29T08:48:12.622612577Z I0729 08:48:12.622592 3399 services_controller.go:244] Starting controller ovn-lb-controller for network=default 2026-07-29T08:48:12.622641393Z I0729 08:48:12.622630 3399 services_controller.go:256] Waiting for node handler to sync for network=default 2026-07-29T08:48:12.622661694Z I0729 08:48:12.622647 3399 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" 2026-07-29T08:48:12.622733496Z I0729 08:48:12.622716 3399 node_info.go:82] Node ip-10-0-141-66.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-141-66.ec2.internal" 2026-07-29T08:48:12.622749650Z I0729 08:48:12.622739 3399 services_controller.go:1038] Node ip-10-0-141-66.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:12.622797802Z I0729 08:48:12.622787 3399 services_controller.go:1038] Node ip-10-0-131-230.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:12.622827110Z I0729 08:48:12.622817 3399 services_controller.go:1038] Node ip-10-0-133-190.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:12.627848615Z + for port_name in $(ovs-vsctl list-ports "$bridge") 2026-07-29T08:48:12.627848615Z + [[ ens5 == *to-br-int ]] 2026-07-29T08:48:12.627864219Z + return 1 2026-07-29T08:48:12.627888198Z + (( retries += 1 )) 2026-07-29T08:48:12.627911432Z + [[ 2 -eq 15 ]] 2026-07-29T08:48:12.627918587Z + sleep 2 2026-07-29T08:48:12.637861755Z I0729 08:48:12.637821 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.637915873Z E0729 08:48:12.637907 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.678232602Z I0729 08:48:12.678193 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.678265204Z E0729 08:48:12.678258 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.723340866Z I0729 08:48:12.723294 3399 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" 2026-07-29T08:48:12.723340866Z I0729 08:48:12.723320 3399 services_controller.go:264] Setting up event handlers for services for network=default 2026-07-29T08:48:12.723410339Z I0729 08:48:12.723391 3399 services_controller.go:274] Setting up event handlers for endpoint slices for network=default 2026-07-29T08:48:12.723491678Z I0729 08:48:12.723474 3399 services_controller.go:284] Waiting for service and endpoint handlers to sync for network=default 2026-07-29T08:48:12.723512813Z I0729 08:48:12.723496 3399 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" 2026-07-29T08:48:12.758690378Z I0729 08:48:12.758642 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.758754288Z E0729 08:48:12.758738 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.823828145Z I0729 08:48:12.823778 3399 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" 2026-07-29T08:48:12.823828145Z I0729 08:48:12.823806 3399 repair.go:63] Starting repairing loop for services 2026-07-29T08:48:12.823947878Z I0729 08:48:12.823933 3399 repair.go:134] Deleted 0 stale service LBs 2026-07-29T08:48:12.823958461Z I0729 08:48:12.823953 3399 repair.go:140] Deleted 0 stale Chassis Template Vars 2026-07-29T08:48:12.823983373Z I0729 08:48:12.823970 3399 repair.go:65] Finished repairing loop for services: 165.907µs 2026-07-29T08:48:12.824013979Z I0729 08:48:12.824002 3399 services_controller.go:687] Controller cache of 0 load balancers initialized for 0 services for network=default 2026-07-29T08:48:12.824110618Z I0729 08:48:12.824093 3399 services_controller.go:311] Starting workers for network=default 2026-07-29T08:48:12.824622886Z I0729 08:48:12.824593 3399 base_network_controller_pods.go:490] [default/kube-system/global-pull-secret-syncer-8t5kw] creating logical port kube-system_global-pull-secret-syncer-8t5kw for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.824699404Z I0729 08:48:12.824597 3399 base_network_controller_pods.go:490] [default/openshift-multus/network-metrics-daemon-js5hz] creating logical port openshift-multus_network-metrics-daemon-js5hz for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.824767484Z I0729 08:48:12.824748 3399 base_network_controller_pods.go:490] [default/openshift-network-diagnostics/network-check-target-4q6x4] creating logical port openshift-network-diagnostics_network-check-target-4q6x4 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.824818801Z I0729 08:48:12.824792 3399 kube.go:234] Patching annotations on pod openshift-multus/network-metrics-daemon-js5hz 2026-07-29T08:48:12.824870389Z I0729 08:48:12.824760 3399 kube.go:234] Patching annotations on pod kube-system/global-pull-secret-syncer-8t5kw 2026-07-29T08:48:12.824932679Z I0729 08:48:12.824915 3399 kube.go:234] Patching annotations on pod openshift-network-diagnostics/network-check-target-4q6x4 2026-07-29T08:48:12.836162379Z I0729 08:48:12.836127 3399 base_network_controller_pods.go:976] [default/kube-system/global-pull-secret-syncer-8t5kw] addLogicalPort annotation time took 11.451779ms 2026-07-29T08:48:12.837299109Z I0729 08:48:12.837266 3399 pods.go:273] [kube-system/global-pull-secret-syncer-8t5kw] addLogicalPort took 12.690319ms, libovsdb time 811.978µs 2026-07-29T08:48:12.845271966Z I0729 08:48:12.845243 3399 base_network_controller_pods.go:976] [default/openshift-network-diagnostics/network-check-target-4q6x4] addLogicalPort annotation time took 20.373502ms 2026-07-29T08:48:12.845303534Z I0729 08:48:12.845265 3399 base_network_controller_pods.go:976] [default/openshift-multus/network-metrics-daemon-js5hz] addLogicalPort annotation time took 20.506534ms 2026-07-29T08:48:12.846092621Z I0729 08:48:12.846069 3399 pods.go:273] [openshift-multus/network-metrics-daemon-js5hz] addLogicalPort took 21.568412ms, libovsdb time 606.839µs 2026-07-29T08:48:12.846469377Z I0729 08:48:12.846437 3399 pods.go:273] [openshift-network-diagnostics/network-check-target-4q6x4] addLogicalPort took 21.694166ms, libovsdb time 927.311µs 2026-07-29T08:48:12.846808478Z I0729 08:48:12.846784 3399 admin_network_policy_controller.go:231] Starting controller default-network-controller 2026-07-29T08:48:12.846838636Z I0729 08:48:12.846821 3399 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" 2026-07-29T08:48:12.846847808Z I0729 08:48:12.846841 3399 shared_informer.go:356] "Caches are synced" controller="default-network-controller" 2026-07-29T08:48:12.846854869Z I0729 08:48:12.846847 3399 admin_network_policy_controller.go:241] Repairing Admin Network Policies 2026-07-29T08:48:12.846982270Z I0729 08:48:12.846969 3399 repair.go:32] Repairing admin network policies took 115.693µs 2026-07-29T08:48:12.847050068Z I0729 08:48:12.847028 3399 repair.go:95] Repairing baseline admin network policies took 49.447µs 2026-07-29T08:48:12.847581515Z I0729 08:48:12.847567 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default 2026-07-29T08:48:12.847605046Z I0729 08:48:12.847594 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] 2026-07-29T08:48:12.849306339Z I0729 08:48:12.849287 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel 2026-07-29T08:48:12.849324944Z I0729 08:48:12.849315 3399 base_network_controller_policy.go:1190] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default 2026-07-29T08:48:12.849341829Z I0729 08:48:12.849333 3399 base_network_controller_policy.go:1085] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] 2026-07-29T08:48:12.851510038Z I0729 08:48:12.851488 3399 base_network_controller_policy.go:1243] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel 2026-07-29T08:48:12.851527369Z I0729 08:48:12.851519 3399 base_network_controller_policy.go:1190] Adding network policy openshift-operators/default-allow-all for network default 2026-07-29T08:48:12.851536111Z I0729 08:48:12.851529 3399 base_network_controller_policy.go:1085] Policy openshift-operators/default-allow-all added to peer address sets [] 2026-07-29T08:48:12.853405615Z I0729 08:48:12.853390 3399 base_network_controller_policy.go:1243] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel 2026-07-29T08:48:12.853419102Z I0729 08:48:12.853413 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default 2026-07-29T08:48:12.853427405Z I0729 08:48:12.853422 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] 2026-07-29T08:48:12.855193244Z I0729 08:48:12.855173 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel 2026-07-29T08:48:12.855205091Z I0729 08:48:12.855196 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default 2026-07-29T08:48:12.855211922Z I0729 08:48:12.855205 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] 2026-07-29T08:48:12.855833742Z I0729 08:48:12.855816 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel 2026-07-29T08:48:12.855846878Z I0729 08:48:12.855831 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default 2026-07-29T08:48:12.855846878Z I0729 08:48:12.855839 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] 2026-07-29T08:48:12.856457457Z I0729 08:48:12.856439 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel 2026-07-29T08:48:12.856473090Z I0729 08:48:12.856457 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cloud-credential-operator/default-deny for network default 2026-07-29T08:48:12.856473090Z I0729 08:48:12.856465 3399 base_network_controller_policy.go:1085] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] 2026-07-29T08:48:12.857841485Z I0729 08:48:12.857823 3399 base_network_controller_policy.go:1243] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel 2026-07-29T08:48:12.857858806Z I0729 08:48:12.857840 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-version/default-deny for network default 2026-07-29T08:48:12.857858806Z I0729 08:48:12.857848 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-version/default-deny added to peer address sets [] 2026-07-29T08:48:12.859206791Z I0729 08:48:12.859189 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel 2026-07-29T08:48:12.859223410Z I0729 08:48:12.859207 3399 base_network_controller_policy.go:1190] Adding network policy openshift-marketplace/unpack-bundles for network default 2026-07-29T08:48:12.859223410Z I0729 08:48:12.859219 3399 base_network_controller_policy.go:1085] Policy openshift-marketplace/unpack-bundles added to peer address sets [] 2026-07-29T08:48:12.861107983Z I0729 08:48:12.861088 3399 base_network_controller_policy.go:1243] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel 2026-07-29T08:48:12.861124914Z I0729 08:48:12.861111 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default 2026-07-29T08:48:12.861550249Z I0729 08:48:12.861530 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] 2026-07-29T08:48:12.862367645Z I0729 08:48:12.862349 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel 2026-07-29T08:48:12.862383944Z I0729 08:48:12.862369 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default 2026-07-29T08:48:12.862392737Z I0729 08:48:12.862386 3399 base_network_controller_policy.go:1085] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] 2026-07-29T08:48:12.862979356Z I0729 08:48:12.862960 3399 base_network_controller_policy.go:1243] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel 2026-07-29T08:48:12.862999159Z I0729 08:48:12.862984 3399 base_network_controller_policy.go:1190] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default 2026-07-29T08:48:12.862999159Z I0729 08:48:12.862991 3399 base_network_controller_policy.go:1085] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] 2026-07-29T08:48:12.863407207Z I0729 08:48:12.863386 3399 base_network_controller_policy.go:1243] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel 2026-07-29T08:48:12.863424228Z I0729 08:48:12.863409 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default 2026-07-29T08:48:12.863433694Z I0729 08:48:12.863421 3399 base_network_controller_policy.go:1085] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] 2026-07-29T08:48:12.864048402Z I0729 08:48:12.864030 3399 base_network_controller_policy.go:1243] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel 2026-07-29T08:48:12.864065172Z I0729 08:48:12.864049 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-storage-operator/default-deny for network default 2026-07-29T08:48:12.864065172Z I0729 08:48:12.864056 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] 2026-07-29T08:48:12.864458294Z I0729 08:48:12.864437 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel 2026-07-29T08:48:12.864474858Z I0729 08:48:12.864460 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default 2026-07-29T08:48:12.864484237Z I0729 08:48:12.864474 3399 base_network_controller_policy.go:1085] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] 2026-07-29T08:48:12.865104858Z I0729 08:48:12.865086 3399 base_network_controller_policy.go:1243] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel 2026-07-29T08:48:12.865122058Z I0729 08:48:12.865104 3399 base_network_controller_policy.go:1190] Adding network policy openshift-marketplace/default-deny-all for network default 2026-07-29T08:48:12.865122058Z I0729 08:48:12.865110 3399 base_network_controller_policy.go:1085] Policy openshift-marketplace/default-deny-all added to peer address sets [] 2026-07-29T08:48:12.865427429Z I0729 08:48:12.865411 3399 base_network_controller_policy.go:1243] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel 2026-07-29T08:48:12.865450499Z I0729 08:48:12.865440 3399 base_network_controller_policy.go:1190] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default 2026-07-29T08:48:12.865466605Z I0729 08:48:12.865455 3399 base_network_controller_policy.go:1085] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] 2026-07-29T08:48:12.866952602Z I0729 08:48:12.866935 3399 base_network_controller_policy.go:1243] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel 2026-07-29T08:48:12.866971630Z I0729 08:48:12.866951 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default 2026-07-29T08:48:12.866981895Z I0729 08:48:12.866974 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] 2026-07-29T08:48:12.867688417Z I0729 08:48:12.867669 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel 2026-07-29T08:48:12.867702700Z I0729 08:48:12.867697 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default 2026-07-29T08:48:12.867709519Z I0729 08:48:12.867704 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] 2026-07-29T08:48:12.868294856Z I0729 08:48:12.868277 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel 2026-07-29T08:48:12.868309753Z I0729 08:48:12.868296 3399 base_network_controller_policy.go:1190] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default 2026-07-29T08:48:12.868309753Z I0729 08:48:12.868306 3399 base_network_controller_policy.go:1085] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] 2026-07-29T08:48:12.869674576Z I0729 08:48:12.869654 3399 base_network_controller_policy.go:1243] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel 2026-07-29T08:48:12.869694575Z I0729 08:48:12.869675 3399 base_network_controller_policy.go:1190] Adding network policy openshift-marketplace/marketplace-operator for network default 2026-07-29T08:48:12.869694575Z I0729 08:48:12.869684 3399 base_network_controller_policy.go:1085] Policy openshift-marketplace/marketplace-operator added to peer address sets [] 2026-07-29T08:48:12.870854394Z I0729 08:48:12.870831 3399 base_network_controller_policy.go:1243] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel 2026-07-29T08:48:12.870876200Z I0729 08:48:12.870854 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default 2026-07-29T08:48:12.870876200Z I0729 08:48:12.870865 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] 2026-07-29T08:48:12.871422959Z I0729 08:48:12.871404 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel 2026-07-29T08:48:12.871446243Z I0729 08:48:12.871425 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default 2026-07-29T08:48:12.871446243Z I0729 08:48:12.871434 3399 base_network_controller_policy.go:1085] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] 2026-07-29T08:48:12.871993213Z I0729 08:48:12.871975 3399 base_network_controller_policy.go:1243] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel 2026-07-29T08:48:12.871993213Z I0729 08:48:12.871990 3399 base_network_controller_policy.go:1190] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default 2026-07-29T08:48:12.872011378Z I0729 08:48:12.871999 3399 base_network_controller_policy.go:1085] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] 2026-07-29T08:48:12.873422531Z I0729 08:48:12.873399 3399 base_network_controller_policy.go:1243] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel 2026-07-29T08:48:12.873444600Z I0729 08:48:12.873422 3399 base_network_controller_policy.go:1190] Adding network policy openshift-cloud-credential-operator/allow-egress for network default 2026-07-29T08:48:12.873444600Z I0729 08:48:12.873433 3399 base_network_controller_policy.go:1085] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] 2026-07-29T08:48:12.874243349Z I0729 08:48:12.874225 3399 base_network_controller_policy.go:1243] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel 2026-07-29T08:48:12.874270399Z I0729 08:48:12.874259 3399 controller.go:177] Starting controller default-network-controller-egressip-nad with 1 workers 2026-07-29T08:48:12.879271927Z I0729 08:48:12.879249 3399 egressfirewall.go:369] Starting EgressFirewall controller 2026-07-29T08:48:12.879293452Z I0729 08:48:12.879285 3399 controller.go:153] Adding controller egress-firewall-controller event handlers 2026-07-29T08:48:12.879304350Z I0729 08:48:12.879287 3399 controller.go:153] Adding controller egress-firewall-controller-node event handlers 2026-07-29T08:48:12.879320418Z I0729 08:48:12.879312 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" 2026-07-29T08:48:12.879329231Z I0729 08:48:12.879322 3399 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" 2026-07-29T08:48:12.879338026Z I0729 08:48:12.879325 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" 2026-07-29T08:48:12.879346919Z I0729 08:48:12.879340 3399 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" 2026-07-29T08:48:12.879526070Z I0729 08:48:12.879511 3399 controller.go:177] Starting controller egress-firewall-controller with 1 workers 2026-07-29T08:48:12.879537933Z I0729 08:48:12.879527 3399 controller.go:177] Starting controller egress-firewall-controller-node with 1 workers 2026-07-29T08:48:12.879537933Z I0729 08:48:12.879535 3399 controller.go:177] Starting controller egress-firewall-controller-NAD with 1 workers 2026-07-29T08:48:12.879559470Z I0729 08:48:12.879548 3399 egressqos.go:188] Setting up event handlers for EgressQoS 2026-07-29T08:48:12.879619225Z I0729 08:48:12.879606 3399 egressfirewall.go:710] Syncing node "ip-10-0-141-66.ec2.internal" for egress firewall 2026-07-29T08:48:12.879619225Z I0729 08:48:12.879616 3399 egressfirewall.go:710] Syncing node "ip-10-0-131-230.ec2.internal" for egress firewall 2026-07-29T08:48:12.879628888Z I0729 08:48:12.879620 3399 egressfirewall.go:710] Syncing node "ip-10-0-133-190.ec2.internal" for egress firewall 2026-07-29T08:48:12.879699502Z I0729 08:48:12.879686 3399 egressqos.go:240] Starting EgressQoS Controller 2026-07-29T08:48:12.879710273Z I0729 08:48:12.879699 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" 2026-07-29T08:48:12.879719436Z I0729 08:48:12.879709 3399 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" 2026-07-29T08:48:12.879728411Z I0729 08:48:12.879718 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" 2026-07-29T08:48:12.879737283Z I0729 08:48:12.879727 3399 shared_informer.go:356] "Caches are synced" controller="egressqospods" 2026-07-29T08:48:12.879746182Z I0729 08:48:12.879736 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" 2026-07-29T08:48:12.879755299Z I0729 08:48:12.879744 3399 shared_informer.go:356] "Caches are synced" controller="egressqos" 2026-07-29T08:48:12.879755299Z I0729 08:48:12.879751 3399 egressqos.go:254] Repairing EgressQoSes 2026-07-29T08:48:12.879764297Z I0729 08:48:12.879757 3399 egressqos.go:394] Starting repairing loop for egressqos 2026-07-29T08:48:12.879801683Z I0729 08:48:12.879788 3399 egressqos.go:396] Finished repairing loop for egressqos: 32.709µs 2026-07-29T08:48:12.879821883Z I0729 08:48:12.879811 3399 egressservice_zone.go:135] Setting up event handlers for Egress Services 2026-07-29T08:48:12.879884130Z I0729 08:48:12.879869 3399 egressqos.go:999] Processing sync for EgressQoS node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.879943446Z I0729 08:48:12.879930 3399 egressqos.go:1002] Finished syncing EgressQoS node ip-10-0-133-190.ec2.internal : 62.915µs 2026-07-29T08:48:12.880034832Z I0729 08:48:12.880021 3399 egressservice_zone.go:213] Starting Egress Services Controller 2026-07-29T08:48:12.880045122Z I0729 08:48:12.880033 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" 2026-07-29T08:48:12.880045122Z I0729 08:48:12.880039 3399 shared_informer.go:356] "Caches are synced" controller="egressservices" 2026-07-29T08:48:12.880058510Z I0729 08:48:12.880045 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" 2026-07-29T08:48:12.880058510Z I0729 08:48:12.880050 3399 shared_informer.go:356] "Caches are synced" controller="egressservices_services" 2026-07-29T08:48:12.880068129Z I0729 08:48:12.880056 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" 2026-07-29T08:48:12.880068129Z I0729 08:48:12.880061 3399 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" 2026-07-29T08:48:12.880075103Z I0729 08:48:12.880069 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" 2026-07-29T08:48:12.880081615Z I0729 08:48:12.880076 3399 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" 2026-07-29T08:48:12.880088107Z I0729 08:48:12.880082 3399 egressservice_zone.go:231] Repairing Egress Services 2026-07-29T08:48:12.880188140Z I0729 08:48:12.880175 3399 master_controller.go:91] Starting Admin Policy Based Route Controller 2026-07-29T08:48:12.880188140Z I0729 08:48:12.880184 3399 external_controller.go:282] Starting Admin Policy Based Route Controller 2026-07-29T08:48:12.880199882Z I0729 08:48:12.880194 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-131-230.ec2.internal 2026-07-29T08:48:12.880216041Z I0729 08:48:12.880205 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-131-230.ec2.internal: 12.127µs 2026-07-29T08:48:12.880216041Z I0729 08:48:12.880214 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:12.880223477Z I0729 08:48:12.880218 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-133-190.ec2.internal: 4.45µs 2026-07-29T08:48:12.880230095Z I0729 08:48:12.880223 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.880230095Z I0729 08:48:12.880226 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-141-66.ec2.internal: 3.818µs 2026-07-29T08:48:12.880289661Z I0729 08:48:12.880277 3399 default_network_controller.go:830] Completing all the Watchers took 304.875985ms 2026-07-29T08:48:12.880298864Z I0729 08:48:12.880289 3399 default_network_controller.go:834] Starting unidling controllers 2026-07-29T08:48:12.880298864Z I0729 08:48:12.880295 3399 unidle.go:49] Registering OVN SB ControllerEvent handler 2026-07-29T08:48:12.880307457Z I0729 08:48:12.880300 3399 unidle.go:66] Populating Initial ContollerEvent events 2026-07-29T08:48:12.880318494Z I0729 08:48:12.880311 3399 unidle.go:82] Setting up event handlers for services 2026-07-29T08:48:12.880403245Z I0729 08:48:12.880390 3399 ovnkube.go:548] Waiting for OVN northbound database changes to be processed by ovn-controller 2026-07-29T08:48:12.919074296Z I0729 08:48:12.919029 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:12.919126348Z E0729 08:48:12.919108 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:12.994688083Z I0729 08:48:12.994653 3399 default_node_network_controller.go:549] Node connection status = connected 2026-07-29T08:48:13.239627112Z I0729 08:48:13.239591 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:13.239694728Z E0729 08:48:13.239679 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:13.321456231Z I0729 08:48:13.321410 3399 ovnkube.go:553] Finished waiting for OVN northbound database changes to be processed by ovn-controller 2026-07-29T08:48:13.495199793Z I0729 08:48:13.495161 3399 default_node_network_controller.go:549] Node connection status = connected 2026-07-29T08:48:13.512790884Z I0729 08:48:13.512755 3399 bridgeconfig.go:551] Gateway is ready 2026-07-29T08:48:13.512790884Z I0729 08:48:13.512787 3399 gateway_shared_intf.go:1787] Creating Gateway Openflow Manager 2026-07-29T08:48:13.524172370Z I0729 08:48:13.524138 3399 node_ip_handler_linux.go:318] Node primary address changed to 10.0.133.190. Updating OVN encap IP. 2026-07-29T08:48:13.524199947Z I0729 08:48:13.524172 3399 node_ip_handler_linux.go:632] Will not update encap IP 10.0.133.190 - it is already configured 2026-07-29T08:48:13.524199947Z I0729 08:48:13.524192 3399 kube.go:266] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.133.190"]] on node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.532398670Z I0729 08:48:13.532365 3399 node_ip_handler_linux.go:581] Node address changed to map[10.0.133.190/20:{} 172.20.0.1/32:{}]. Updating annotations. 2026-07-29T08:48:13.532617359Z I0729 08:48:13.532592 3399 kube.go:266] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.133.190/20","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-133-190.ec2.internal","mac-address":"02:98:b3:77:59:bf","ip-addresses":["10.0.133.190/20"],"ip-address":"10.0.133.190/20","next-hops":["10.0.128.1"],"next-hop":"10.0.128.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3b78e20e-60b0-44ab-80ad-255ec40ceb5f k8s.ovn.org/node-encap-ips:["10.0.133.190"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.133.190/20"}] on node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.544446165Z I0729 08:48:13.544416 3399 gateway_shared_intf.go:2084] Setting OVN Masquerade route with source: 10.0.133.190 2026-07-29T08:48:13.544502357Z I0729 08:48:13.544480 3399 kube.go:266] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.544928108Z I0729 08:48:13.544887 3399 egressfirewall.go:710] Syncing node "ip-10-0-133-190.ec2.internal" for egress firewall 2026-07-29T08:48:13.544972091Z I0729 08:48:13.544933 3399 default_network_controller.go:1124] Egress IP detected IP address change for node ip-10-0-133-190.ec2.internal. Updating no re-route policies 2026-07-29T08:48:13.545038504Z I0729 08:48:13.545005 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.545038504Z I0729 08:48:13.545029 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-133-190.ec2.internal: 27.747µs 2026-07-29T08:48:13.545097204Z I0729 08:48:13.545061 3399 services_controller.go:1038] Node ip-10-0-133-190.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:13.545097204Z I0729 08:48:13.545081 3399 master.go:559] Adding or Updating local node "ip-10-0-133-190.ec2.internal" for network "default" 2026-07-29T08:48:13.550149333Z I0729 08:48:13.550117 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-l2js5 2026-07-29T08:48:13.550149333Z I0729 08:48:13.550129 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-js5hz 2026-07-29T08:48:13.550149333Z I0729 08:48:13.550138 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-network-operator/iptables-alerter-l2js5 2026-07-29T08:48:13.550149333Z I0729 08:48:13.550145 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-multus/network-metrics-daemon-js5hz 2026-07-29T08:48:13.550180155Z I0729 08:48:13.550146 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-image-registry/node-ca-4f49z 2026-07-29T08:48:13.550180155Z I0729 08:48:13.550161 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-image-registry/node-ca-4f49z 2026-07-29T08:48:13.550180155Z I0729 08:48:13.550168 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550180155Z I0729 08:48:13.550164 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550180155Z I0729 08:48:13.550174 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550180155Z I0729 08:48:13.550176 3399 base_network_controller_pods.go:490] [default/openshift-multus/network-metrics-daemon-js5hz] creating logical port openshift-multus_network-metrics-daemon-js5hz for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550193079Z I0729 08:48:13.550184 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550193079Z I0729 08:48:13.550185 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod kube-system/konnectivity-agent-fdhtb 2026-07-29T08:48:13.550202618Z I0729 08:48:13.550188 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ccppr 2026-07-29T08:48:13.550202618Z I0729 08:48:13.550196 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-gcj42 2026-07-29T08:48:13.550202618Z I0729 08:48:13.550198 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod kube-system/konnectivity-agent-fdhtb 2026-07-29T08:48:13.550212684Z I0729 08:48:13.550199 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-multus/multus-bnntk 2026-07-29T08:48:13.550212684Z I0729 08:48:13.550203 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-gcj42 2026-07-29T08:48:13.550222021Z I0729 08:48:13.550210 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-multus/multus-bnntk 2026-07-29T08:48:13.550222021Z I0729 08:48:13.550213 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-4q6x4 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550219 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-8t5kw 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550222 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-4q6x4 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550224 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-dns/node-resolver-7fnsg 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550228 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod kube-system/global-pull-secret-syncer-8t5kw 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550232 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-dns/node-resolver-7fnsg 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550174 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-45rbx 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550243 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-45rbx 2026-07-29T08:48:13.550249169Z I0729 08:48:13.550245 3399 base_network_controller_pods.go:490] [default/openshift-network-diagnostics/network-check-target-4q6x4] creating logical port openshift-network-diagnostics_network-check-target-4q6x4 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550260294Z I0729 08:48:13.550201 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ccppr 2026-07-29T08:48:13.550269161Z I0729 08:48:13.550189 3399 obj_retry.go:362] default/networkController: retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fzlp6 2026-07-29T08:48:13.550269161Z I0729 08:48:13.550263 3399 obj_retry.go:426] default/networkController: adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fzlp6 2026-07-29T08:48:13.550269161Z I0729 08:48:13.550262 3399 base_network_controller_pods.go:490] [default/kube-system/global-pull-secret-syncer-8t5kw] creating logical port kube-system_global-pull-secret-syncer-8t5kw for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:13.550858847Z I0729 08:48:13.550834 3399 pods.go:273] [kube-system/global-pull-secret-syncer-8t5kw] addLogicalPort took 585.055µs, libovsdb time 245.669µs 2026-07-29T08:48:13.550945957Z I0729 08:48:13.550922 3399 pods.go:273] [openshift-network-diagnostics/network-check-target-4q6x4] addLogicalPort took 683.933µs, libovsdb time 256.853µs 2026-07-29T08:48:13.551055251Z I0729 08:48:13.551039 3399 pods.go:273] [openshift-multus/network-metrics-daemon-js5hz] addLogicalPort took 872.364µs, libovsdb time 368.625µs 2026-07-29T08:48:13.557036162Z I0729 08:48:13.557008 3399 gateway_shared_intf.go:1852] Creating Gateway Node Port Watcher 2026-07-29T08:48:13.569453470Z I0729 08:48:13.569427 3399 iptables.go:111] Creating table: mangle chain: OUTPUT 2026-07-29T08:48:13.572499233Z I0729 08:48:13.572483 3399 iptables.go:111] Creating table: nat chain: OUTPUT 2026-07-29T08:48:13.592544786Z I0729 08:48:13.592506 3399 iptables.go:111] Creating table: nat chain: PREROUTING 2026-07-29T08:48:13.685985317Z I0729 08:48:13.685949 3399 iptables.go:111] Creating table: filter chain: FORWARD 2026-07-29T08:48:13.702903979Z I0729 08:48:13.702860 3399 gateway_shared_intf.go:2275] Ensuring IP Neighbor entry for: 169.254.0.1 2026-07-29T08:48:13.703014637Z W0729 08:48:13.703002 3399 gateway_shared_intf.go:2281] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory 2026-07-29T08:48:13.703087729Z I0729 08:48:13.703078 3399 gateway_shared_intf.go:2275] Ensuring IP Neighbor entry for: 169.254.0.4 2026-07-29T08:48:13.703206911Z W0729 08:48:13.703196 3399 gateway_shared_intf.go:2281] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory 2026-07-29T08:48:13.703269118Z I0729 08:48:13.703255 3399 gateway_init.go:333] Gateway and management port readiness took 1.213395722s 2026-07-29T08:48:13.703291085Z I0729 08:48:13.703281 3399 controller.go:153] Adding controller [node-nad-controller NAD controller] event handlers 2026-07-29T08:48:13.703327887Z I0729 08:48:13.703314 3399 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" 2026-07-29T08:48:13.703336936Z I0729 08:48:13.703327 3399 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" 2026-07-29T08:48:13.703345659Z I0729 08:48:13.703335 3399 nad_controller.go:971] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes 2026-07-29T08:48:13.703399612Z I0729 08:48:13.703389 3399 controller.go:177] Starting controller [node-nad-controller NAD controller] with 1 workers 2026-07-29T08:48:13.703497225Z I0729 08:48:13.703481 3399 nad_controller.go:1010] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.792µs 2026-07-29T08:48:13.703729774Z I0729 08:48:13.703719 3399 network_controller.go:376] [node-nad-controller network controller]: syncing all networks 2026-07-29T08:48:13.703753026Z I0729 08:48:13.703742 3399 network_controller.go:401] [node-nad-controller network controller]: finished syncing network default, took 15.283µs 2026-07-29T08:48:13.703753026Z I0729 08:48:13.703749 3399 network_controller.go:383] [node-nad-controller network controller]: finished syncing all networks. Time taken: 32.034µs 2026-07-29T08:48:13.703760743Z I0729 08:48:13.703755 3399 controller.go:177] Starting controller [node-nad-controller network controller] with 1 workers 2026-07-29T08:48:13.703786234Z I0729 08:48:13.703777 3399 nad_controller.go:552] [node-nad-controller NAD controller]: started 2026-07-29T08:48:13.703795131Z I0729 08:48:13.703789 3399 default_node_network_controller.go:943] Starting the default node network controller 2026-07-29T08:48:13.703801643Z I0729 08:48:13.703794 3399 gateway_init.go:343] Initializing Gateway Functionality for gateway Start 2026-07-29T08:48:13.703863344Z I0729 08:48:13.703851 3399 network_controller.go:401] [node-nad-controller network controller]: finished syncing network default, took 12.866µs 2026-07-29T08:48:13.703999478Z I0729 08:48:13.703987 3399 default_node_network_controller.go:1539] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). 2026-07-29T08:48:13.703999478Z I0729 08:48:13.703996 3399 default_node_network_controller.go:979] Waiting for gateway and management port readiness... 2026-07-29T08:48:13.704063866Z I0729 08:48:13.704019 3399 obj_retry_gateway.go:36] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b0ce30 0xc000247dc0 0xc000247e30 0xc000247f10 0xc00027c000 0xc00027c070 0xc00027c0e0 0xc000b3b310 0xc00027c150 0xc00027c1c0 0xc000d061e0 0xc00617e2a0 0xc00027c230 map[0x307b8c0:0xc002598000 0x307be00:0xc002598180 0x307cb20:0xc00219fce0 0x307cdc0:0xc00219fd40 0x307d5a0:0xc002598060 0x307d840:0xc0025980c0 0x307dae0:0xc002598120 0x307e2c0:0xc0025981e0 0x307e560:0xc002598240 0x309f6a0:0xc000df2ba0 0x30a10a0:0xc000d34000 0x30a1a60:0xc000760a20 0x30a1da0:0xc0007609c0 0x30a2de0:0xc00219fc80 0x30a3460:0xc0008d41e0] 0xc00089a460 0} 2026-07-29T08:48:13.704075517Z I0729 08:48:13.704070 3399 gateway.go:156] Starting gateway service sync 2026-07-29T08:48:13.704493890Z I0729 08:48:13.704482 3399 gateway_iptables.go:477] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP 2026-07-29T08:48:13.706559085Z I0729 08:48:13.706542 3399 gateway_iptables.go:477] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT 2026-07-29T08:48:13.729065040Z I0729 08:48:13.729029 3399 gateway_iptables.go:477] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP 2026-07-29T08:48:13.730261730Z I0729 08:48:13.730247 3399 gateway_iptables.go:477] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP 2026-07-29T08:48:13.731438049Z I0729 08:48:13.731417 3399 gateway_iptables.go:477] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP 2026-07-29T08:48:13.766198173Z I0729 08:48:13.766155 3399 gateway.go:173] Gateway service sync done. Time taken: 62.072759ms 2026-07-29T08:48:13.766910822Z I0729 08:48:13.766876 3399 healthcheck.go:148] Opening healthcheck "openshift-ingress/router-default" on port 31243 2026-07-29T08:48:13.767038998Z I0729 08:48:13.766997 3399 obj_retry_gateway.go:36] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b0ce30 0xc000247dc0 0xc000247e30 0xc000247f10 0xc00027c000 0xc00027c070 0xc00027c0e0 0xc000b3b310 0xc00027c150 0xc00027c1c0 0xc000d061e0 0xc00617e2a0 0xc00027c230 map[0x307b8c0:0xc002598000 0x307be00:0xc002598180 0x307cb20:0xc00219fce0 0x307cdc0:0xc00219fd40 0x307d5a0:0xc002598060 0x307d840:0xc0025980c0 0x307dae0:0xc002598120 0x307e2c0:0xc0025981e0 0x307e560:0xc002598240 0x309f6a0:0xc000df2ba0 0x30a10a0:0xc000d34000 0x30a1a60:0xc000760a20 0x30a1da0:0xc0007609c0 0x30a2de0:0xc00219fc80 0x30a3460:0xc0008d41e0] 0xc00089a460 0} 2026-07-29T08:48:13.767524573Z I0729 08:48:13.767475 3399 obj_retry_gateway.go:36] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b0ce30 0xc000247dc0 0xc000247e30 0xc000247f10 0xc00027c000 0xc00027c070 0xc00027c0e0 0xc000b3b310 0xc00027c150 0xc00027c1c0 0xc000d061e0 0xc00617e2a0 0xc00027c230 map[0x307b8c0:0xc002598000 0x307be00:0xc002598180 0x307cb20:0xc00219fce0 0x307cdc0:0xc00219fd40 0x307d5a0:0xc002598060 0x307d840:0xc0025980c0 0x307dae0:0xc002598120 0x307e2c0:0xc0025981e0 0x307e560:0xc002598240 0x309f6a0:0xc000df2ba0 0x30a10a0:0xc000d34000 0x30a1a60:0xc000760a20 0x30a1da0:0xc0007609c0 0x30a2de0:0xc00219fc80 0x30a3460:0xc0008d41e0] 0xc00089a460 0} 2026-07-29T08:48:13.767544672Z I0729 08:48:13.767536 3399 gateway.go:535] Reconciling gateway with updates 2026-07-29T08:48:13.773034266Z I0729 08:48:13.773005 3399 gateway.go:342] Spawning Conntrack Rule Check Thread 2026-07-29T08:48:13.773065713Z I0729 08:48:13.773035 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics 2026-07-29T08:48:13.773065713Z I0729 08:48:13.773046 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift 2026-07-29T08:48:13.773065713Z I0729 08:48:13.773055 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics 2026-07-29T08:48:13.773077086Z I0729 08:48:13.773063 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/openshift 2026-07-29T08:48:13.773077086Z I0729 08:48:13.773066 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-config-operator/metrics 2026-07-29T08:48:13.773092638Z I0729 08:48:13.773077 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller 2026-07-29T08:48:13.773092638Z I0729 08:48:13.773078 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-config-operator/metrics 2026-07-29T08:48:13.773092638Z I0729 08:48:13.773086 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller 2026-07-29T08:48:13.773102672Z I0729 08:48:13.773089 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns/dns-default 2026-07-29T08:48:13.773112225Z I0729 08:48:13.773104 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns/dns-default 2026-07-29T08:48:13.773112225Z I0729 08:48:13.773061 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator 2026-07-29T08:48:13.773124043Z I0729 08:48:13.773115 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator 2026-07-29T08:48:13.773150496Z I0729 08:48:13.773035 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio 2026-07-29T08:48:13.773160719Z I0729 08:48:13.773148 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway open-cluster-management-agent-addon/cluster-proxy-601dbc94553ef959f1f3923b0ca941357a1b65468c936d42d 2026-07-29T08:48:13.773160719Z I0729 08:48:13.773155 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio 2026-07-29T08:48:13.773169530Z I0729 08:48:13.773087 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver 2026-07-29T08:48:13.773178133Z I0729 08:48:13.773167 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway open-cluster-management-agent-addon/cluster-proxy-601dbc94553ef959f1f3923b0ca941357a1b65468c936d42d 2026-07-29T08:48:13.773178133Z I0729 08:48:13.773174 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver 2026-07-29T08:48:13.773189160Z I0729 08:48:13.773171 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics 2026-07-29T08:48:13.773197962Z I0729 08:48:13.773187 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service 2026-07-29T08:48:13.773197962Z I0729 08:48:13.773191 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics 2026-07-29T08:48:13.773208903Z I0729 08:48:13.773200 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service 2026-07-29T08:48:13.773217857Z I0729 08:48:13.773208 3399 default_node_network_controller.go:1004] Gateway and management port readiness took 69.202756ms 2026-07-29T08:48:13.773217857Z I0729 08:48:13.773149 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator 2026-07-29T08:48:13.773229020Z I0729 08:48:13.773118 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon 2026-07-29T08:48:13.773229020Z I0729 08:48:13.773224 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator 2026-07-29T08:48:13.773241097Z I0729 08:48:13.773228 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics 2026-07-29T08:48:13.773241097Z I0729 08:48:13.773231 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon 2026-07-29T08:48:13.773241097Z I0729 08:48:13.773234 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics 2026-07-29T08:48:13.773250480Z I0729 08:48:13.773239 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics 2026-07-29T08:48:13.773250480Z I0729 08:48:13.773242 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics 2026-07-29T08:48:13.773260035Z I0729 08:48:13.773248 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics 2026-07-29T08:48:13.773260035Z I0729 08:48:13.773252 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics 2026-07-29T08:48:13.773260035Z I0729 08:48:13.773255 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-insights/metrics 2026-07-29T08:48:13.773288838Z I0729 08:48:13.773266 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-default 2026-07-29T08:48:13.773299555Z I0729 08:48:13.773288 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/packageserver 2026-07-29T08:48:13.773299555Z I0729 08:48:13.773291 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver 2026-07-29T08:48:13.773308661Z I0729 08:48:13.773296 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service 2026-07-29T08:48:13.773308661Z I0729 08:48:13.773295 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-console-operator/metrics 2026-07-29T08:48:13.773308661Z I0729 08:48:13.773231 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server 2026-07-29T08:48:13.773318269Z I0729 08:48:13.773308 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service 2026-07-29T08:48:13.773318269Z I0729 08:48:13.773310 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics 2026-07-29T08:48:13.773318269Z I0729 08:48:13.773309 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-console-operator/metrics 2026-07-29T08:48:13.773327774Z I0729 08:48:13.773314 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/kubernetes 2026-07-29T08:48:13.773327774Z I0729 08:48:13.773317 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics 2026-07-29T08:48:13.773336815Z I0729 08:48:13.773326 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/kubernetes 2026-07-29T08:48:13.773348143Z I0729 08:48:13.773106 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-operator/metrics 2026-07-29T08:48:13.773357334Z I0729 08:48:13.773349 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-operator/metrics 2026-07-29T08:48:13.773357334Z I0729 08:48:13.773350 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics 2026-07-29T08:48:13.773367298Z I0729 08:48:13.773178 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-apiserver 2026-07-29T08:48:13.773367298Z I0729 08:48:13.773364 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry 2026-07-29T08:48:13.773376695Z I0729 08:48:13.773368 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-apiserver 2026-07-29T08:48:13.773376695Z I0729 08:48:13.773370 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-image-registry/image-registry 2026-07-29T08:48:13.773385699Z I0729 08:48:13.773074 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target 2026-07-29T08:48:13.773394560Z I0729 08:48:13.773386 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics 2026-07-29T08:48:13.773394560Z I0729 08:48:13.773387 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target 2026-07-29T08:48:13.773403827Z I0729 08:48:13.773390 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics 2026-07-29T08:48:13.773413069Z I0729 08:48:13.773404 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics 2026-07-29T08:48:13.773438186Z I0729 08:48:13.773054 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics 2026-07-29T08:48:13.773438186Z I0729 08:48:13.773433 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics 2026-07-29T08:48:13.773450786Z I0729 08:48:13.773444 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics 2026-07-29T08:48:13.773486520Z I0729 08:48:13.773473 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator 2026-07-29T08:48:13.773494089Z I0729 08:48:13.773487 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator 2026-07-29T08:48:13.773500553Z I0729 08:48:13.773313 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server 2026-07-29T08:48:13.773549140Z I0729 08:48:13.773441 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns-operator/metrics 2026-07-29T08:48:13.773582467Z I0729 08:48:13.773133 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node 2026-07-29T08:48:13.773582467Z I0729 08:48:13.773573 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node 2026-07-29T08:48:13.773582467Z I0729 08:48:13.773066 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin 2026-07-29T08:48:13.773600291Z I0729 08:48:13.773584 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin 2026-07-29T08:48:13.773600291Z I0729 08:48:13.773159 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics 2026-07-29T08:48:13.773611575Z I0729 08:48:13.773599 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics 2026-07-29T08:48:13.773640222Z I0729 08:48:13.773214 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary 2026-07-29T08:48:13.773651574Z I0729 08:48:13.773642 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary 2026-07-29T08:48:13.773697274Z I0729 08:48:13.773223 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default 2026-07-29T08:48:13.773707875Z I0729 08:48:13.773694 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default 2026-07-29T08:48:13.773750578Z I0729 08:48:13.773268 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-insights/metrics 2026-07-29T08:48:13.773771799Z I0729 08:48:13.773275 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints 2026-07-29T08:48:13.773781247Z I0729 08:48:13.773771 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints 2026-07-29T08:48:13.773804523Z I0729 08:48:13.773276 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics 2026-07-29T08:48:13.773804523Z I0729 08:48:13.773801 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics 2026-07-29T08:48:13.773830441Z I0729 08:48:13.773297 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-default 2026-07-29T08:48:13.773843600Z I0729 08:48:13.773836 3399 port_claim.go:74] Svc openshift-ingress/router-default: port 31178 is already open, no action needed 2026-07-29T08:48:13.773852825Z I0729 08:48:13.773845 3399 port_claim.go:74] Svc openshift-ingress/router-default: port 32190 is already open, no action needed 2026-07-29T08:48:13.773938618Z I0729 08:48:13.773299 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/packageserver 2026-07-29T08:48:13.774054895Z I0729 08:48:13.773302 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver 2026-07-29T08:48:13.774054895Z I0729 08:48:13.773338 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics 2026-07-29T08:48:13.774085946Z I0729 08:48:13.774054 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics 2026-07-29T08:48:13.774085946Z I0729 08:48:13.773376 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source 2026-07-29T08:48:13.774085946Z I0729 08:48:13.774065 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source 2026-07-29T08:48:13.774449745Z I0729 08:48:13.774434 3399 gateway_shared_intf.go:2225] Ensuring masquerade resources on br-ex (ifindex=5, lastSuccessful=0) 2026-07-29T08:48:13.774794614Z I0729 08:48:13.774781 3399 gateway_shared_intf.go:2084] Setting OVN Masquerade route with source: 10.0.133.190 2026-07-29T08:48:13.775012338Z I0729 08:48:13.774999 3399 gateway_shared_intf.go:2275] Ensuring IP Neighbor entry for: 169.254.0.1 2026-07-29T08:48:13.775089924Z I0729 08:48:13.775078 3399 gateway_shared_intf.go:2275] Ensuring IP Neighbor entry for: 169.254.0.4 2026-07-29T08:48:13.775178025Z I0729 08:48:13.775166 3399 gateway_shared_intf.go:2251] Masquerade resources ensured on br-ex (ifindex=5) 2026-07-29T08:48:13.775233352Z I0729 08:48:13.775222 3399 node_ip_handler_linux.go:197] Node IP manager is running 2026-07-29T08:48:13.814183423Z W0729 08:48:13.814111 3399 port_linux.go:278] Missing or unable to find route entry for subnet 10.132.0.0/14 via gateway 10.133.0.1 on link ovn-k8s-mp0 with MTU: 0 2026-07-29T08:48:13.814441620Z W0729 08:48:13.814417 3399 port_linux.go:278] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.133.0.1 on link ovn-k8s-mp0 with MTU: 0 2026-07-29T08:48:13.814593949Z W0729 08:48:13.814566 3399 port_linux.go:294] Missing arp entry for MAC/IP binding (0a:58:0a:85:00:01/10.133.0.1) on link ovn-k8s-mp0 2026-07-29T08:48:13.880307090Z I0729 08:48:13.880265 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:13.880358670Z E0729 08:48:13.880333 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:14.629621552Z + [[ 2 -lt 90 ]] 2026-07-29T08:48:14.629621552Z + add_garp_drop_flow br-ex 2026-07-29T08:48:14.629656963Z + local bridge=br-ex 2026-07-29T08:48:14.629656963Z + local cookie=0x0305 2026-07-29T08:48:14.629664645Z + local priority=499 2026-07-29T08:48:14.630032785Z ++ ovs-vsctl list-ports br-ex 2026-07-29T08:48:14.636424230Z + for port_name in $(ovs-vsctl list-ports "$bridge") 2026-07-29T08:48:14.636424230Z + [[ ens5 == *to-br-int ]] 2026-07-29T08:48:14.636445031Z + for port_name in $(ovs-vsctl list-ports "$bridge") 2026-07-29T08:48:14.636445031Z + [[ patch-br-ex_ip-10-0-133-190.ec2.internal-to-br-int == *to-br-int ]] 2026-07-29T08:48:14.636445031Z + local of_port 2026-07-29T08:48:14.636752046Z ++ ovs-vsctl get interface patch-br-ex_ip-10-0-133-190.ec2.internal-to-br-int ofport 2026-07-29T08:48:14.642763936Z + of_port=2 2026-07-29T08:48:14.642784462Z + ovs-ofctl add-flow br-ex cookie=0x0305,table=0,priority=499,in_port=2,arp,arp_op=1,actions=drop 2026-07-29T08:48:14.648933725Z ++ date -Iseconds 2026-07-29T08:48:14.650288666Z + echo '2026-07-29T08:48:14+00:00 - INFO - Added GARP drop flow on br-ex port patch-br-ex_ip-10-0-133-190.ec2.internal-to-br-int (ofport=2)' 2026-07-29T08:48:14.650299930Z 2026-07-29T08:48:14+00:00 - INFO - Added GARP drop flow on br-ex port patch-br-ex_ip-10-0-133-190.ec2.internal-to-br-int (ofport=2) 2026-07-29T08:48:14.650311475Z + return 0 2026-07-29T08:48:14.650331436Z + return 0 2026-07-29T08:48:14.897066042Z I0729 08:48:14.897029 3399 default_node_network_controller.go:1459] Starting node controller node sync 2026-07-29T08:48:14.897066042Z W0729 08:48:14.897054 3399 egressip_healthcheck.go:79] Health checking using insecure connection 2026-07-29T08:48:14.897159148Z I0729 08:48:14.897121 3399 egressip_healthcheck.go:112] Starting Egress IP Health Server on 10.133.0.2:9107 2026-07-29T08:48:14.910327449Z I0729 08:48:14.910281 3399 default_node_network_controller.go:1501] Node controller node sync done. Time taken: 13.231559ms 2026-07-29T08:48:14.910412278Z I0729 08:48:14.910392 3399 default_node_network_controller.go:1372] Adding remote node "ip-10-0-131-230.ec2.internal", IP: 10.0.131.230 to PMTUD blocking rules 2026-07-29T08:48:14.910412278Z I0729 08:48:14.910407 3399 default_node_network_controller.go:1372] Adding remote node "ip-10-0-131-230.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules 2026-07-29T08:48:14.916741964Z I0729 08:48:14.916697 3399 healthcheck_node.go:127] "Starting node proxy healthz server" address="0.0.0.0:10256" 2026-07-29T08:48:14.917238574Z I0729 08:48:14.917208 3399 egressservice_node.go:97] Setting up event handlers for Egress Services 2026-07-29T08:48:14.917391392Z I0729 08:48:14.917359 3399 egressservice_node.go:187] Starting Egress Services Controller 2026-07-29T08:48:14.917391392Z I0729 08:48:14.917382 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" 2026-07-29T08:48:14.917408848Z I0729 08:48:14.917394 3399 shared_informer.go:356] "Caches are synced" controller="egressservices" 2026-07-29T08:48:14.917408848Z I0729 08:48:14.917403 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" 2026-07-29T08:48:14.917418655Z I0729 08:48:14.917411 3399 shared_informer.go:356] "Caches are synced" controller="egressservices_services" 2026-07-29T08:48:14.917428045Z I0729 08:48:14.917417 3399 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" 2026-07-29T08:48:14.917428045Z I0729 08:48:14.917421 3399 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" 2026-07-29T08:48:14.917428045Z I0729 08:48:14.917425 3399 egressservice_node.go:201] Repairing Egress Services 2026-07-29T08:48:14.929449272Z I0729 08:48:14.929415 3399 node_controller.go:46] Starting Admin Policy Based Route Node Controller 2026-07-29T08:48:14.929449272Z I0729 08:48:14.929431 3399 external_controller.go:282] Starting Admin Policy Based Route Controller 2026-07-29T08:48:14.929640084Z I0729 08:48:14.929607 3399 default_node_network_controller.go:1156] Egress IP for secondary host network is disabled 2026-07-29T08:48:14.929640084Z I0729 08:48:14.929628 3399 link_network_manager.go:122] Link manager is running 2026-07-29T08:48:14.929662309Z I0729 08:48:14.929646 3399 default_node_network_controller.go:1177] Default node network controller initialized and ready. 2026-07-29T08:48:14.929878860Z I0729 08:48:14.929858 3399 ovspinning_linux.go:53] OVS CPU affinity pinning disabled 2026-07-29T08:48:14.930107771Z I0729 08:48:14.930092 3399 vrf_manager.go:139] VRF manager is running 2026-07-29T08:48:14.930245128Z I0729 08:48:14.930223 3399 node_controller_manager.go:469] Removing flows to drop GARP 2026-07-29T08:48:14.930258727Z I0729 08:48:14.930242 3399 gateway.go:535] Reconciling gateway with updates 2026-07-29T08:48:14.942908340Z I0729 08:48:14.942851 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/packageserver 2026-07-29T08:48:14.942908340Z I0729 08:48:14.942879 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics 2026-07-29T08:48:14.942953039Z I0729 08:48:14.942880 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node 2026-07-29T08:48:14.942953039Z I0729 08:48:14.942915 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics 2026-07-29T08:48:14.942953039Z I0729 08:48:14.942919 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-default 2026-07-29T08:48:14.942953039Z I0729 08:48:14.942934 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics 2026-07-29T08:48:14.942953039Z I0729 08:48:14.942934 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator 2026-07-29T08:48:14.942966230Z I0729 08:48:14.942948 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift 2026-07-29T08:48:14.942966230Z I0729 08:48:14.942953 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/kubernetes 2026-07-29T08:48:14.942966230Z I0729 08:48:14.942960 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/openshift 2026-07-29T08:48:14.942976332Z I0729 08:48:14.942963 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/kubernetes 2026-07-29T08:48:14.942976332Z I0729 08:48:14.942965 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default 2026-07-29T08:48:14.942976332Z I0729 08:48:14.942934 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-default 2026-07-29T08:48:14.942986000Z I0729 08:48:14.942973 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source 2026-07-29T08:48:14.942994994Z I0729 08:48:14.942988 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target 2026-07-29T08:48:14.942994994Z I0729 08:48:14.942990 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source 2026-07-29T08:48:14.943004402Z I0729 08:48:14.942995 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target 2026-07-29T08:48:14.943004402Z I0729 08:48:14.942952 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator 2026-07-29T08:48:14.943013811Z I0729 08:48:14.942995 3399 port_claim.go:74] Svc openshift-ingress/router-default: port 31178 is already open, no action needed 2026-07-29T08:48:14.943013811Z I0729 08:48:14.943008 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon 2026-07-29T08:48:14.943023059Z I0729 08:48:14.943015 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon 2026-07-29T08:48:14.943023059Z I0729 08:48:14.943019 3399 port_claim.go:74] Svc openshift-ingress/router-default: port 32190 is already open, no action needed 2026-07-29T08:48:14.943036884Z I0729 08:48:14.943027 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator 2026-07-29T08:48:14.943048047Z I0729 08:48:14.943042 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator 2026-07-29T08:48:14.943056862Z I0729 08:48:14.942919 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service 2026-07-29T08:48:14.943065626Z I0729 08:48:14.943055 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service 2026-07-29T08:48:14.943065626Z I0729 08:48:14.943056 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns/dns-default 2026-07-29T08:48:14.943074445Z I0729 08:48:14.943063 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller 2026-07-29T08:48:14.943074445Z I0729 08:48:14.943068 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns/dns-default 2026-07-29T08:48:14.943074445Z I0729 08:48:14.943069 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics 2026-07-29T08:48:14.943084625Z I0729 08:48:14.943074 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller 2026-07-29T08:48:14.943093292Z I0729 08:48:14.943081 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics 2026-07-29T08:48:14.943104625Z I0729 08:48:14.942858 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics 2026-07-29T08:48:14.943113598Z I0729 08:48:14.942882 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/packageserver 2026-07-29T08:48:14.943113598Z I0729 08:48:14.942974 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default 2026-07-29T08:48:14.943122994Z I0729 08:48:14.943112 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics 2026-07-29T08:48:14.943134301Z I0729 08:48:14.942919 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node 2026-07-29T08:48:14.943160750Z I0729 08:48:14.942922 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics 2026-07-29T08:48:14.943171823Z I0729 08:48:14.943157 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver 2026-07-29T08:48:14.943171823Z I0729 08:48:14.943162 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio 2026-07-29T08:48:14.943180855Z I0729 08:48:14.942854 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics 2026-07-29T08:48:14.943189090Z I0729 08:48:14.943177 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio 2026-07-29T08:48:14.943189090Z I0729 08:48:14.943180 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics 2026-07-29T08:48:14.943204117Z I0729 08:48:14.942940 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin 2026-07-29T08:48:14.943215409Z I0729 08:48:14.943206 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin 2026-07-29T08:48:14.943223776Z I0729 08:48:14.943211 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints 2026-07-29T08:48:14.943232240Z I0729 08:48:14.943223 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server 2026-07-29T08:48:14.943240646Z I0729 08:48:14.943234 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server 2026-07-29T08:48:14.943250014Z I0729 08:48:14.943238 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry 2026-07-29T08:48:14.943250014Z I0729 08:48:14.943247 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-image-registry/image-registry 2026-07-29T08:48:14.943259007Z I0729 08:48:14.943169 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver 2026-07-29T08:48:14.943269565Z I0729 08:48:14.943146 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics 2026-07-29T08:48:14.943278927Z I0729 08:48:14.943266 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service 2026-07-29T08:48:14.943278927Z I0729 08:48:14.943223 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints 2026-07-29T08:48:14.943278927Z I0729 08:48:14.943275 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service 2026-07-29T08:48:14.943288814Z I0729 08:48:14.943274 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics 2026-07-29T08:48:14.943318894Z I0729 08:48:14.943303 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator 2026-07-29T08:48:14.943318894Z I0729 08:48:14.943311 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-apiserver 2026-07-29T08:48:14.943329098Z I0729 08:48:14.943317 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator 2026-07-29T08:48:14.943329098Z I0729 08:48:14.943320 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-apiserver 2026-07-29T08:48:14.943337861Z I0729 08:48:14.943325 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics 2026-07-29T08:48:14.943337861Z I0729 08:48:14.943330 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-insights/metrics 2026-07-29T08:48:14.943347111Z I0729 08:48:14.943339 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-insights/metrics 2026-07-29T08:48:14.943347111Z I0729 08:48:14.943321 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics 2026-07-29T08:48:14.943359432Z I0729 08:48:14.943348 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics 2026-07-29T08:48:14.943359432Z I0729 08:48:14.943352 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics 2026-07-29T08:48:14.943359432Z I0729 08:48:14.943353 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns-operator/metrics 2026-07-29T08:48:14.943369251Z I0729 08:48:14.943361 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics 2026-07-29T08:48:14.943369251Z I0729 08:48:14.943361 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics 2026-07-29T08:48:14.943381420Z I0729 08:48:14.943373 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary 2026-07-29T08:48:14.943390605Z I0729 08:48:14.943381 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics 2026-07-29T08:48:14.943390605Z I0729 08:48:14.943384 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary 2026-07-29T08:48:14.943399657Z I0729 08:48:14.943392 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics 2026-07-29T08:48:14.943408575Z I0729 08:48:14.943398 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway open-cluster-management-agent-addon/cluster-proxy-601dbc94553ef959f1f3923b0ca941357a1b65468c936d42d 2026-07-29T08:48:14.943417178Z I0729 08:48:14.943362 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics 2026-07-29T08:48:14.943417178Z I0729 08:48:14.943409 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver 2026-07-29T08:48:14.943426344Z I0729 08:48:14.943418 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver 2026-07-29T08:48:14.943426344Z I0729 08:48:14.943421 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-config-operator/metrics 2026-07-29T08:48:14.943435422Z I0729 08:48:14.943423 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics 2026-07-29T08:48:14.943435422Z I0729 08:48:14.943430 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-config-operator/metrics 2026-07-29T08:48:14.943453149Z I0729 08:48:14.943437 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics 2026-07-29T08:48:14.943462268Z I0729 08:48:14.943410 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway open-cluster-management-agent-addon/cluster-proxy-601dbc94553ef959f1f3923b0ca941357a1b65468c936d42d 2026-07-29T08:48:14.943481088Z I0729 08:48:14.943466 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-console-operator/metrics 2026-07-29T08:48:14.943490230Z I0729 08:48:14.943480 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-console-operator/metrics 2026-07-29T08:48:14.943510407Z I0729 08:48:14.943498 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics 2026-07-29T08:48:14.943510407Z I0729 08:48:14.943508 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics 2026-07-29T08:48:14.943519952Z I0729 08:48:14.943469 3399 obj_retry.go:362] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-operator/metrics 2026-07-29T08:48:14.943530984Z I0729 08:48:14.943523 3399 obj_retry.go:426] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-operator/metrics 2026-07-29T08:48:15.160491390Z I0729 08:48:15.160447 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:15.160549333Z E0729 08:48:15.160534 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:17.406514221Z I0729 08:48:17.406466 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-133-190.ec2.internal 2026-07-29T08:48:17.406577004Z I0729 08:48:17.406506 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-133-190.ec2.internal: 47.966µs 2026-07-29T08:48:17.491226270Z I0729 08:48:17.491170 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:17.491265332Z E0729 08:48:17.491243 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:17.499905009Z I0729 08:48:17.499842 3399 base_network_controller_pods.go:490] [default/openshift-dns/dns-default-sg6gw] creating logical port openshift-dns_dns-default-sg6gw for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:17.500022429Z I0729 08:48:17.499995 3399 kube.go:234] Patching annotations on pod openshift-dns/dns-default-sg6gw 2026-07-29T08:48:17.500573905Z I0729 08:48:17.500546 3399 base_network_controller_pods.go:490] [default/openshift-ingress-canary/ingress-canary-zcdl7] creating logical port openshift-ingress-canary_ingress-canary-zcdl7 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:17.500698871Z I0729 08:48:17.500681 3399 kube.go:234] Patching annotations on pod openshift-ingress-canary/ingress-canary-zcdl7 2026-07-29T08:48:17.525171198Z I0729 08:48:17.525132 3399 base_network_controller_pods.go:976] [default/openshift-ingress-canary/ingress-canary-zcdl7] addLogicalPort annotation time took 24.484073ms 2026-07-29T08:48:17.526301420Z I0729 08:48:17.526272 3399 pods.go:273] [openshift-ingress-canary/ingress-canary-zcdl7] addLogicalPort took 25.736582ms, libovsdb time 879.676µs 2026-07-29T08:48:17.539494443Z I0729 08:48:17.539459 3399 base_network_controller_pods.go:976] [default/openshift-dns/dns-default-sg6gw] addLogicalPort annotation time took 39.497701ms 2026-07-29T08:48:17.541062430Z I0729 08:48:17.541030 3399 pods.go:273] [openshift-dns/dns-default-sg6gw] addLogicalPort took 41.203681ms, libovsdb time 879.286µs 2026-07-29T08:48:17.721458262Z I0729 08:48:17.721417 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:17.721511161Z E0729 08:48:17.721495 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:17.880209036Z I0729 08:48:17.880158 3399 cniserver.go:311] [openshift-ingress-canary/ingress-canary-zcdl7 b308d26ace8521d424f8d7c273c8433a18fe6f9321d80692e731e2bb80cf55f9 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:17.901947340Z I0729 08:48:17.901587 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-zcdl7, hostIfaceName: b308d26ace8521d, network: default, NAD default, SandboxID: "b308d26ace8521d424f8d7c273c8433a18fe6f9321d80692e731e2bb80cf55f9", PCI device ID: , UID: "44283aff-2b70-482c-ab25-f91bd9e7713c", MAC: 0a:58:0a:85:00:07, IPs: [10.133.0.7/23] 2026-07-29T08:48:17.960069519Z I0729 08:48:17.960018 3399 cniserver.go:300] [openshift-ingress-canary/ingress-canary-zcdl7 b308d26ace8521d424f8d7c273c8433a18fe6f9321d80692e731e2bb80cf55f9 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-zcdl7 b308d26ace8521d424f8d7c273c8433a18fe6f9321d80692e731e2bb80cf55f9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:38:44:96:e1:be\",\"name\":\"b308d26ace8521d\"},{\"mac\":\"0a:58:0a:85:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e56bab7c-48fe-4b88-896c-e327f49e2b64\"}],\"ips\":[{\"address\":\"10.133.0.7/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:27.382589664Z I0729 08:48:27.382548 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:27.382634490Z E0729 08:48:27.382610 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:27.523924788Z I0729 08:48:27.523863 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:27.523965714Z E0729 08:48:27.523934 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:27.962383795Z I0729 08:48:27.962340 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:27.962420269Z E0729 08:48:27.962393 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:31.297045747Z I0729 08:48:31.296993 3399 egressfirewall.go:710] Syncing node "ip-10-0-141-66.ec2.internal" for egress firewall 2026-07-29T08:48:31.297092516Z I0729 08:48:31.297044 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:31.297092516Z I0729 08:48:31.297063 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-141-66.ec2.internal: 21.257µs 2026-07-29T08:48:31.297092516Z I0729 08:48:31.297072 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up 2026-07-29T08:48:31.297173426Z E0729 08:48:31.297154 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:35.991505165Z I0729 08:48:35.991461 3399 node_info.go:82] Node ip-10-0-141-66.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-141-66.ec2.internal" 2026-07-29T08:48:35.991547521Z I0729 08:48:35.991501 3399 services_controller.go:1038] Node ip-10-0-141-66.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:35.991547521Z I0729 08:48:35.991508 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "ip-10-0-141-66.ec2.internal", network "default", needs interconnect zone sync up 2026-07-29T08:48:35.991567667Z E0729 08:48:35.991550 3399 controller.go:278] Controller node-topology-node: error found while processing ip-10-0-141-66.ec2.internal|default: adding or updating remote node chassis ip-10-0-141-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-141-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-141-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:36.129817945Z I0729 08:48:36.129773 3399 default_network_controller.go:564] Node "ip-10-0-141-66.ec2.internal" in remote zone "ip-10-0-141-66.ec2.internal", network "default", needs interconnect zone sync up 2026-07-29T08:48:36.129881310Z I0729 08:48:36.129778 3399 services_controller.go:1038] Node ip-10-0-141-66.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:36.130799376Z I0729 08:48:36.130764 3399 zone_ic_handler.go:198] Time taken to create transit switch: 317.246µs 2026-07-29T08:48:36.130799376Z I0729 08:48:36.130786 3399 zone_ic_handler.go:280] Creating interconnect resources for remote zone node ip-10-0-141-66.ec2.internal for the network default 2026-07-29T08:48:36.131887239Z I0729 08:48:36.131859 3399 zone_ic_handler.go:285] Creating Interconnect resources for node "ip-10-0-141-66.ec2.internal" on network "default" took: 1.430771ms 2026-07-29T08:48:38.687027616Z I0729 08:48:38.686980 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:38.687027616Z I0729 08:48:38.687009 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-141-66.ec2.internal: 34.603µs 2026-07-29T08:48:38.687100522Z I0729 08:48:38.687077 3399 egressfirewall.go:710] Syncing node "ip-10-0-141-66.ec2.internal" for egress firewall 2026-07-29T08:48:38.687100522Z I0729 08:48:38.687085 3399 services_controller.go:1038] Node ip-10-0-141-66.ec2.internal switch + router changed, syncing services in network "default" 2026-07-29T08:48:38.687111927Z I0729 08:48:38.687094 3399 default_node_network_controller.go:1372] Adding remote node "ip-10-0-141-66.ec2.internal", IP: 10.0.141.66 to PMTUD blocking rules 2026-07-29T08:48:38.687111927Z I0729 08:48:38.687105 3399 default_node_network_controller.go:1372] Adding remote node "ip-10-0-141-66.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules 2026-07-29T08:48:38.693408009Z I0729 08:48:38.693374 3399 default_network_controller.go:1124] Egress IP detected IP address change for node ip-10-0-141-66.ec2.internal. Updating no re-route policies 2026-07-29T08:48:38.708019998Z I0729 08:48:38.707987 3399 default_node_network_controller.go:1372] Adding remote node "ip-10-0-141-66.ec2.internal", IP: 10.0.141.66 to PMTUD blocking rules 2026-07-29T08:48:38.708077814Z I0729 08:48:38.708062 3399 default_node_network_controller.go:1372] Adding remote node "ip-10-0-141-66.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules 2026-07-29T08:48:42.744254221Z I0729 08:48:42.744212 3399 egressservice_zone_node.go:114] Processing sync for Egress Service node ip-10-0-141-66.ec2.internal 2026-07-29T08:48:42.744254221Z I0729 08:48:42.744243 3399 egressservice_zone_node.go:117] Finished syncing Egress Service node ip-10-0-141-66.ec2.internal: 38.727µs 2026-07-29T08:48:43.052705250Z W0729 08:48:43.052663 3399 util.go:881] Failed to build global endpoints for port TCP//8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target 2026-07-29T08:48:48.697805011Z I0729 08:48:48.697765 3399 cniserver.go:311] [kube-system/global-pull-secret-syncer-8t5kw 70f968c50fa3f0d4e73fae6421015f71e5fdcb89dea2261cf5e43fb5e0fcbb01 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:48.709337001Z I0729 08:48:48.709289 3399 helper_linux.go:533] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-8t5kw, hostIfaceName: 70f968c50fa3f0d, network: default, NAD default, SandboxID: "70f968c50fa3f0d4e73fae6421015f71e5fdcb89dea2261cf5e43fb5e0fcbb01", PCI device ID: , UID: "888bb542-8e91-474c-a2c6-9ef7f46aff79", MAC: 0a:58:0a:85:00:03, IPs: [10.133.0.3/23] 2026-07-29T08:48:48.725122034Z I0729 08:48:48.725081 3399 cniserver.go:311] [openshift-network-diagnostics/network-check-target-4q6x4 3496ec6b1e698b5fb4309a5cbd2aef5eb8b9b9f0655d6e190c092aa49ea580c6 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:48.735626957Z I0729 08:48:48.735588 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-4q6x4, hostIfaceName: 3496ec6b1e698b5, network: default, NAD default, SandboxID: "3496ec6b1e698b5fb4309a5cbd2aef5eb8b9b9f0655d6e190c092aa49ea580c6", PCI device ID: , UID: "279be192-0ab2-4804-86d7-0c7abb990291", MAC: 0a:58:0a:85:00:05, IPs: [10.133.0.5/23] 2026-07-29T08:48:48.766831762Z I0729 08:48:48.766771 3399 cniserver.go:300] [kube-system/global-pull-secret-syncer-8t5kw 70f968c50fa3f0d4e73fae6421015f71e5fdcb89dea2261cf5e43fb5e0fcbb01 network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-8t5kw 70f968c50fa3f0d4e73fae6421015f71e5fdcb89dea2261cf5e43fb5e0fcbb01 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:20:d7:8f:ff:12\",\"name\":\"70f968c50fa3f0d\"},{\"mac\":\"0a:58:0a:85:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78a6f1d8-1ab7-489e-9398-d7145b163cd5\"}],\"ips\":[{\"address\":\"10.133.0.3/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:48.786787570Z I0729 08:48:48.786740 3399 cniserver.go:300] [openshift-network-diagnostics/network-check-target-4q6x4 3496ec6b1e698b5fb4309a5cbd2aef5eb8b9b9f0655d6e190c092aa49ea580c6 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-4q6x4 3496ec6b1e698b5fb4309a5cbd2aef5eb8b9b9f0655d6e190c092aa49ea580c6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:45:77:37:28:7b\",\"name\":\"3496ec6b1e698b5\"},{\"mac\":\"0a:58:0a:85:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d97aad61-9f93-4ad0-a003-37d459d02eb6\"}],\"ips\":[{\"address\":\"10.133.0.5/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:50.008776815Z I0729 08:48:50.008734 3399 base_network_controller_pods.go:490] [default/openshift-console-operator/console-operator-584dcb6b54-wdbhh] creating logical port openshift-console-operator_console-operator-584dcb6b54-wdbhh for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:50.009036211Z I0729 08:48:50.008999 3399 kube.go:234] Patching annotations on pod openshift-console-operator/console-operator-584dcb6b54-wdbhh 2026-07-29T08:48:50.023838234Z I0729 08:48:50.023795 3399 base_network_controller_pods.go:976] [default/openshift-console-operator/console-operator-584dcb6b54-wdbhh] addLogicalPort annotation time took 14.828134ms 2026-07-29T08:48:50.025740362Z I0729 08:48:50.025710 3399 pods.go:273] [openshift-console-operator/console-operator-584dcb6b54-wdbhh] addLogicalPort took 16.996513ms, libovsdb time 1.398266ms 2026-07-29T08:48:50.114348822Z I0729 08:48:50.114307 3399 base_network_controller_pods.go:490] [default/openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-549c6d54cf-w7c9q for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:50.114887430Z I0729 08:48:50.114815 3399 kube.go:234] Patching annotations on pod openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q 2026-07-29T08:48:50.116381284Z I0729 08:48:50.116345 3399 base_network_controller_pods.go:490] [default/openshift-ingress/router-default-68555bcf85-tdf9s] creating logical port openshift-ingress_router-default-68555bcf85-tdf9s for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:50.116640266Z I0729 08:48:50.116622 3399 kube.go:234] Patching annotations on pod openshift-ingress/router-default-68555bcf85-tdf9s 2026-07-29T08:48:50.117164096Z I0729 08:48:50.117136 3399 base_network_controller_pods.go:490] [default/openshift-insights/insights-operator-5885fdb759-zqcq6] creating logical port openshift-insights_insights-operator-5885fdb759-zqcq6 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:50.117287873Z I0729 08:48:50.117272 3399 kube.go:234] Patching annotations on pod openshift-insights/insights-operator-5885fdb759-zqcq6 2026-07-29T08:48:50.131860842Z I0729 08:48:50.131822 3399 base_network_controller_pods.go:976] [default/openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q] addLogicalPort annotation time took 17.158051ms 2026-07-29T08:48:50.134825619Z I0729 08:48:50.134789 3399 base_network_controller_pods.go:976] [default/openshift-ingress/router-default-68555bcf85-tdf9s] addLogicalPort annotation time took 18.278912ms 2026-07-29T08:48:50.135491165Z I0729 08:48:50.134806 3399 base_network_controller_pods.go:976] [default/openshift-insights/insights-operator-5885fdb759-zqcq6] addLogicalPort annotation time took 17.573774ms 2026-07-29T08:48:50.137118075Z I0729 08:48:50.137091 3399 pods.go:273] [openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q] addLogicalPort took 22.841886ms, libovsdb time 4.678714ms 2026-07-29T08:48:50.137980058Z I0729 08:48:50.137957 3399 pods.go:273] [openshift-insights/insights-operator-5885fdb759-zqcq6] addLogicalPort took 20.838329ms, libovsdb time 1.964272ms 2026-07-29T08:48:50.138091065Z I0729 08:48:50.138071 3399 pods.go:273] [openshift-ingress/router-default-68555bcf85-tdf9s] addLogicalPort took 21.740398ms, libovsdb time 2.773648ms 2026-07-29T08:48:50.425111358Z I0729 08:48:50.425071 3399 cniserver.go:311] [openshift-console-operator/console-operator-584dcb6b54-wdbhh de94bfdba10fe427d7967cd5898a763dee160973aff369aeb1c9864720779cb3 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:50.437031992Z I0729 08:48:50.436986 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-584dcb6b54-wdbhh, hostIfaceName: de94bfdba10fe42, network: default, NAD default, SandboxID: "de94bfdba10fe427d7967cd5898a763dee160973aff369aeb1c9864720779cb3", PCI device ID: , UID: "b5a9ee01-32e8-4005-8e26-205b1afcdde0", MAC: 0a:58:0a:85:00:08, IPs: [10.133.0.8/23] 2026-07-29T08:48:50.500612806Z I0729 08:48:50.500556 3399 cniserver.go:300] [openshift-console-operator/console-operator-584dcb6b54-wdbhh de94bfdba10fe427d7967cd5898a763dee160973aff369aeb1c9864720779cb3 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-584dcb6b54-wdbhh de94bfdba10fe427d7967cd5898a763dee160973aff369aeb1c9864720779cb3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:bb:29:e4:a6:41\",\"name\":\"de94bfdba10fe42\"},{\"mac\":\"0a:58:0a:85:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e61a57d-2236-4922-89ad-2785172c3f1d\"}],\"ips\":[{\"address\":\"10.133.0.8/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:50.548951020Z I0729 08:48:50.548852 3399 cniserver.go:311] [openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q 778107c06a19a3247fdcdd81c6e9a8b4c45d87dbdd19bc15f35667c8cfde5e76 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:50.560574328Z I0729 08:48:50.560528 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-549c6d54cf-w7c9q, hostIfaceName: 778107c06a19a32, network: default, NAD default, SandboxID: "778107c06a19a3247fdcdd81c6e9a8b4c45d87dbdd19bc15f35667c8cfde5e76", PCI device ID: , UID: "38b05c3d-d2a4-4361-90e1-37d0ccbfbfe0", MAC: 0a:58:0a:85:00:09, IPs: [10.133.0.9/23] 2026-07-29T08:48:50.568582341Z I0729 08:48:50.568533 3399 cniserver.go:311] [openshift-insights/insights-operator-5885fdb759-zqcq6 bca7b68e1e89f67e6b0cf40893ad457ec8e695cc254a807846a552bba613d468 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:50.581484444Z I0729 08:48:50.581425 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5885fdb759-zqcq6, hostIfaceName: bca7b68e1e89f67, network: default, NAD default, SandboxID: "bca7b68e1e89f67e6b0cf40893ad457ec8e695cc254a807846a552bba613d468", PCI device ID: , UID: "d3553627-ff60-4db8-8abf-a093140c17e7", MAC: 0a:58:0a:85:00:0b, IPs: [10.133.0.11/23] 2026-07-29T08:48:50.627374108Z I0729 08:48:50.627315 3399 cniserver.go:300] [openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q 778107c06a19a3247fdcdd81c6e9a8b4c45d87dbdd19bc15f35667c8cfde5e76 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-549c6d54cf-w7c9q 778107c06a19a3247fdcdd81c6e9a8b4c45d87dbdd19bc15f35667c8cfde5e76 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:69:7c:52:74:5e\",\"name\":\"778107c06a19a32\"},{\"mac\":\"0a:58:0a:85:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fda32fa2-812d-4c8b-b712-bc194a61c399\"}],\"ips\":[{\"address\":\"10.133.0.9/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:50.644304190Z I0729 08:48:50.644180 3399 cniserver.go:300] [openshift-insights/insights-operator-5885fdb759-zqcq6 bca7b68e1e89f67e6b0cf40893ad457ec8e695cc254a807846a552bba613d468 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-5885fdb759-zqcq6 bca7b68e1e89f67e6b0cf40893ad457ec8e695cc254a807846a552bba613d468 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:aa:d7:94:1d:94\",\"name\":\"bca7b68e1e89f67\"},{\"mac\":\"0a:58:0a:85:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3e43795-c03f-4da5-bf4d-554c013f41fa\"}],\"ips\":[{\"address\":\"10.133.0.11/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:53.698115726Z I0729 08:48:53.698070 3399 namespace.go:89] [openshift-kube-storage-version-migrator] adding namespace 2026-07-29T08:48:53.698789363Z I0729 08:48:53.698747 3399 namespace.go:93] [openshift-kube-storage-version-migrator] adding namespace took 582.419µs 2026-07-29T08:48:53.729098511Z I0729 08:48:53.729056 3399 namespace.go:133] [openshift-kube-storage-version-migrator] updating namespace 2026-07-29T08:48:54.750239415Z I0729 08:48:54.750196 3399 base_network_controller_pods.go:490] [default/openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp] creating logical port openshift-kube-storage-version-migrator_migrator-6865c9967-t2chp for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:54.750477827Z I0729 08:48:54.750444 3399 kube.go:234] Patching annotations on pod openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp 2026-07-29T08:48:54.761990944Z I0729 08:48:54.761917 3399 base_network_controller_pods.go:976] [default/openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp] addLogicalPort annotation time took 11.503753ms 2026-07-29T08:48:54.763361203Z I0729 08:48:54.763328 3399 pods.go:273] [openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp] addLogicalPort took 13.152426ms, libovsdb time 1.05707ms 2026-07-29T08:48:55.033488597Z I0729 08:48:55.033444 3399 namespace.go:89] [openshift-service-ca] adding namespace 2026-07-29T08:48:55.034650595Z I0729 08:48:55.034619 3399 namespace.go:93] [openshift-service-ca] adding namespace took 981.817µs 2026-07-29T08:48:55.052345416Z I0729 08:48:55.052306 3399 namespace.go:133] [openshift-service-ca] updating namespace 2026-07-29T08:48:55.123517389Z I0729 08:48:55.123463 3399 cniserver.go:311] [openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp bc6b4e75ec00f0d22dbc679d8355fba885ae4d1e10372f4d92ee2732d1869954 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:48:55.138987592Z I0729 08:48:55.138948 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-6865c9967-t2chp, hostIfaceName: bc6b4e75ec00f0d, network: default, NAD default, SandboxID: "bc6b4e75ec00f0d22dbc679d8355fba885ae4d1e10372f4d92ee2732d1869954", PCI device ID: , UID: "9e29c609-61bf-42f5-b49f-eaddd8d7c7a5", MAC: 0a:58:0a:85:00:0c, IPs: [10.133.0.12/23] 2026-07-29T08:48:55.213772057Z I0729 08:48:55.213722 3399 cniserver.go:300] [openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp bc6b4e75ec00f0d22dbc679d8355fba885ae4d1e10372f4d92ee2732d1869954 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-6865c9967-t2chp bc6b4e75ec00f0d22dbc679d8355fba885ae4d1e10372f4d92ee2732d1869954 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:d2:32:ad:55:97\",\"name\":\"bc6b4e75ec00f0d\"},{\"mac\":\"0a:58:0a:85:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ae79bd5-25e7-444f-bcff-cfbb9d8b3fa9\"}],\"ips\":[{\"address\":\"10.133.0.12/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:48:56.094612442Z W0729 08:48:56.094571 3399 util.go:881] Failed to build global endpoints for port TCP//8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target 2026-07-29T08:48:56.115564858Z W0729 08:48:56.115526 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-console-operator/metrics 2026-07-29T08:48:56.167393248Z I0729 08:48:56.166921 3399 base_network_controller_pods.go:490] [default/openshift-insights/insights-runtime-extractor-j6zx6] creating logical port openshift-insights_insights-runtime-extractor-j6zx6 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:48:56.167393248Z I0729 08:48:56.167100 3399 kube.go:234] Patching annotations on pod openshift-insights/insights-runtime-extractor-j6zx6 2026-07-29T08:48:56.179377171Z I0729 08:48:56.179324 3399 base_network_controller_pods.go:976] [default/openshift-insights/insights-runtime-extractor-j6zx6] addLogicalPort annotation time took 12.27119ms 2026-07-29T08:48:56.180480226Z I0729 08:48:56.180451 3399 pods.go:273] [openshift-insights/insights-runtime-extractor-j6zx6] addLogicalPort took 13.572498ms, libovsdb time 772.842µs 2026-07-29T08:49:03.564470838Z W0729 08:49:03.564424 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-console-operator/metrics 2026-07-29T08:49:12.171174405Z I0729 08:49:12.171130 3399 cniserver.go:311] [openshift-insights/insights-runtime-extractor-j6zx6 70cf75e1f455b37300a26ff9bb6e4d26abe0b682fb0038cf2858419776bfe202 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:12.181433073Z I0729 08:49:12.181382 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-j6zx6, hostIfaceName: 70cf75e1f455b37, network: default, NAD default, SandboxID: "70cf75e1f455b37300a26ff9bb6e4d26abe0b682fb0038cf2858419776bfe202", PCI device ID: , UID: "e8f4f881-b42e-4d37-ad44-1707cc060151", MAC: 0a:58:0a:85:00:0d, IPs: [10.133.0.13/23] 2026-07-29T08:49:12.230174622Z I0729 08:49:12.230121 3399 cniserver.go:300] [openshift-insights/insights-runtime-extractor-j6zx6 70cf75e1f455b37300a26ff9bb6e4d26abe0b682fb0038cf2858419776bfe202 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-j6zx6 70cf75e1f455b37300a26ff9bb6e4d26abe0b682fb0038cf2858419776bfe202 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:c7:12:09:79:db\",\"name\":\"70cf75e1f455b37\"},{\"mac\":\"0a:58:0a:85:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a79ae5d2-35ec-44ea-944a-83df0c438012\"}],\"ips\":[{\"address\":\"10.133.0.13/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:17.862077906Z I0729 08:49:17.862039 3399 base_network_controller_pods.go:490] [default/open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j] creating logical port open-cluster-management-agent-addon_managed-serviceaccount-addon-agent-6c76f9c46-7262j for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:49:17.862288435Z I0729 08:49:17.862269 3399 kube.go:234] Patching annotations on pod open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j 2026-07-29T08:49:17.880282324Z I0729 08:49:17.880236 3399 base_network_controller_pods.go:976] [default/open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j] addLogicalPort annotation time took 18.030434ms 2026-07-29T08:49:17.882136689Z I0729 08:49:17.882093 3399 pods.go:273] [open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j] addLogicalPort took 20.075796ms, libovsdb time 1.472102ms 2026-07-29T08:49:17.892127884Z W0729 08:49:17.892088 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target 2026-07-29T08:49:17.892247248Z W0729 08:49:17.892218 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target 2026-07-29T08:49:17.892882229Z W0729 08:49:17.892845 3399 util.go:881] Failed to build global endpoints for port TCP/5000-tcp/5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:17.894692504Z W0729 08:49:17.894673 3399 util.go:881] Failed to build global endpoints for port TCP/dns-tcp/5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:17.894743018Z W0729 08:49:17.894724 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/9154: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:17.894743018Z W0729 08:49:17.894738 3399 util.go:881] Failed to build global endpoints for port UDP/dns/5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:17.945493902Z W0729 08:49:17.945457 3399 util.go:881] Failed to build global endpoints for port TCP/5000-tcp/5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:18.027329196Z I0729 08:49:18.027288 3399 base_network_controller_pods.go:490] [default/openshift-image-registry/image-registry-74db7d5bf-7zwv5] creating logical port openshift-image-registry_image-registry-74db7d5bf-7zwv5 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:49:18.027846007Z I0729 08:49:18.027805 3399 kube.go:234] Patching annotations on pod openshift-image-registry/image-registry-74db7d5bf-7zwv5 2026-07-29T08:49:18.063050299Z I0729 08:49:18.062995 3399 base_network_controller_pods.go:976] [default/openshift-image-registry/image-registry-74db7d5bf-7zwv5] addLogicalPort annotation time took 35.239845ms 2026-07-29T08:49:18.064467922Z I0729 08:49:18.064430 3399 pods.go:273] [openshift-image-registry/image-registry-74db7d5bf-7zwv5] addLogicalPort took 37.158815ms, libovsdb time 1.041268ms 2026-07-29T08:49:18.178531911Z W0729 08:49:18.178485 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics 2026-07-29T08:49:18.243125584Z I0729 08:49:18.243080 3399 cniserver.go:311] [open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j 2e56084c77af937764a605126c0f9c7910b1474070a482d9b7462d9210942bcc network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:18.254393153Z I0729 08:49:18.254349 3399 helper_linux.go:533] ConfigureOVS: namespace: open-cluster-management-agent-addon, podName: managed-serviceaccount-addon-agent-6c76f9c46-7262j, hostIfaceName: 2e56084c77af937, network: default, NAD default, SandboxID: "2e56084c77af937764a605126c0f9c7910b1474070a482d9b7462d9210942bcc", PCI device ID: , UID: "47be8c64-d928-4341-91b7-c08f6107153e", MAC: 0a:58:0a:85:00:0e, IPs: [10.133.0.14/23] 2026-07-29T08:49:18.313376018Z I0729 08:49:18.313318 3399 cniserver.go:300] [open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j 2e56084c77af937764a605126c0f9c7910b1474070a482d9b7462d9210942bcc network default NAD default NAD key default] ADD finished CNI request [open-cluster-management-agent-addon/managed-serviceaccount-addon-agent-6c76f9c46-7262j 2e56084c77af937764a605126c0f9c7910b1474070a482d9b7462d9210942bcc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:45:a9:73:a4:f5\",\"name\":\"2e56084c77af937\"},{\"mac\":\"0a:58:0a:85:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/417ea5b5-bcd6-44d2-88f2-c89405ea934e\"}],\"ips\":[{\"address\":\"10.133.0.14/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:18.401789731Z I0729 08:49:18.401737 3399 cniserver.go:311] [openshift-image-registry/image-registry-74db7d5bf-7zwv5 013ea9e2605944d80517e48bfbe1493ccc70d072d11250fae5178db7e0e5f595 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:18.413494072Z I0729 08:49:18.413439 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-74db7d5bf-7zwv5, hostIfaceName: 013ea9e2605944d, network: default, NAD default, SandboxID: "013ea9e2605944d80517e48bfbe1493ccc70d072d11250fae5178db7e0e5f595", PCI device ID: , UID: "01e24675-ede7-48ab-a510-dce1d5414e96", MAC: 0a:58:0a:85:00:0f, IPs: [10.133.0.15/23] 2026-07-29T08:49:18.469849453Z I0729 08:49:18.469800 3399 cniserver.go:300] [openshift-image-registry/image-registry-74db7d5bf-7zwv5 013ea9e2605944d80517e48bfbe1493ccc70d072d11250fae5178db7e0e5f595 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-74db7d5bf-7zwv5 013ea9e2605944d80517e48bfbe1493ccc70d072d11250fae5178db7e0e5f595 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:af:e5:d8:82:02\",\"name\":\"013ea9e2605944d\"},{\"mac\":\"0a:58:0a:85:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9dc34604-252c-42a4-9df7-6335624df9d1\"}],\"ips\":[{\"address\":\"10.133.0.15/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:19.147925049Z W0729 08:49:19.147867 3399 util.go:881] Failed to build global endpoints for port TCP/5000-tcp/5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:22.514706323Z I0729 08:49:22.514647 3399 cniserver.go:311] [openshift-ingress/router-default-68555bcf85-tdf9s f0852b1f09847c32462029c69e2c8cccb90c3610acf457d4bd21f1e69083b92e network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:22.514992019Z I0729 08:49:22.514958 3399 cniserver.go:311] [openshift-dns/dns-default-sg6gw 5c80acb763f43a6871884dc2e70300c84aa784be0f04f462bb567b3bf79efd97 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:22.528044239Z I0729 08:49:22.528005 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-dns, podName: dns-default-sg6gw, hostIfaceName: 5c80acb763f43a6, network: default, NAD default, SandboxID: "5c80acb763f43a6871884dc2e70300c84aa784be0f04f462bb567b3bf79efd97", PCI device ID: , UID: "4e13d41b-8be4-437a-9629-443e1e5cf2eb", MAC: 0a:58:0a:85:00:06, IPs: [10.133.0.6/23] 2026-07-29T08:49:22.533509635Z I0729 08:49:22.532047 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: router-default-68555bcf85-tdf9s, hostIfaceName: f0852b1f09847c3, network: default, NAD default, SandboxID: "f0852b1f09847c32462029c69e2c8cccb90c3610acf457d4bd21f1e69083b92e", PCI device ID: , UID: "a4982339-3ff4-407a-80ae-922cfa8cdcbd", MAC: 0a:58:0a:85:00:0a, IPs: [10.133.0.10/23] 2026-07-29T08:49:22.588996553Z I0729 08:49:22.588942 3399 cniserver.go:300] [openshift-dns/dns-default-sg6gw 5c80acb763f43a6871884dc2e70300c84aa784be0f04f462bb567b3bf79efd97 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-sg6gw 5c80acb763f43a6871884dc2e70300c84aa784be0f04f462bb567b3bf79efd97 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:1c:15:af:5e:22\",\"name\":\"5c80acb763f43a6\"},{\"mac\":\"0a:58:0a:85:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/77417dcb-b4a6-443c-957a-56daa6bd395f\"}],\"ips\":[{\"address\":\"10.133.0.6/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:22.610018412Z I0729 08:49:22.609966 3399 cniserver.go:300] [openshift-ingress/router-default-68555bcf85-tdf9s f0852b1f09847c32462029c69e2c8cccb90c3610acf457d4bd21f1e69083b92e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-68555bcf85-tdf9s f0852b1f09847c32462029c69e2c8cccb90c3610acf457d4bd21f1e69083b92e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:1d:07:0d:5e:fc\",\"name\":\"f0852b1f09847c3\"},{\"mac\":\"0a:58:0a:85:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20ecb883-3c82-4614-9310-51c07133351e\"}],\"ips\":[{\"address\":\"10.133.0.10/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:23.196640130Z W0729 08:49:23.196599 3399 util.go:881] Failed to build global endpoints for port TCP/http/80: empty IP address endpoints for service openshift-ingress/router-default 2026-07-29T08:49:23.196640130Z W0729 08:49:23.196626 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/router-default 2026-07-29T08:49:23.202970853Z W0729 08:49:23.202936 3399 util.go:881] Failed to build global endpoints for port TCP/http/80: empty IP address endpoints for service openshift-ingress/router-internal-default 2026-07-29T08:49:23.203027152Z W0729 08:49:23.203015 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/1936: empty IP address endpoints for service openshift-ingress/router-internal-default 2026-07-29T08:49:23.203055110Z W0729 08:49:23.203046 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/router-internal-default 2026-07-29T08:49:24.175531674Z W0729 08:49:24.175481 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default 2026-07-29T08:49:24.175531674Z W0729 08:49:24.175505 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default 2026-07-29T08:49:24.224962163Z I0729 08:49:24.224918 3399 iptables.go:111] Creating table: nat chain: OVN-KUBE-NODEPORT 2026-07-29T08:49:24.240639050Z W0729 08:49:24.240595 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default 2026-07-29T08:49:24.240639050Z W0729 08:49:24.240618 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default 2026-07-29T08:49:24.240639050Z W0729 08:49:24.240627 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default 2026-07-29T08:49:27.063799031Z W0729 08:49:27.063704 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target 2026-07-29T08:49:27.973337994Z W0729 08:49:27.973293 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:27.973413404Z W0729 08:49:27.973394 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:30.754668564Z W0729 08:49:30.754617 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook 2026-07-29T08:49:32.146925270Z W0729 08:49:32.146859 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:32.146925270Z W0729 08:49:32.146909 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:36.736308227Z W0729 08:49:36.736257 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:36.736308227Z W0729 08:49:36.736288 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:36.736308227Z W0729 08:49:36.736297 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:36.736370405Z W0729 08:49:36.736316 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:36.736370405Z W0729 08:49:36.736323 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:36.736370405Z W0729 08:49:36.736330 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:37.194384668Z W0729 08:49:37.194334 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:37.194384668Z W0729 08:49:37.194361 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:37.194384668Z W0729 08:49:37.194369 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default 2026-07-29T08:49:40.154216980Z W0729 08:49:40.154165 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry 2026-07-29T08:49:42.827138730Z W0729 08:49:42.827086 3399 util.go:881] Failed to build global endpoints for port TCP/http/8080: empty IP address endpoints for service openshift-console/downloads 2026-07-29T08:49:42.859185136Z W0729 08:49:42.859144 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-console/console 2026-07-29T08:49:42.922087084Z W0729 08:49:42.922046 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-console/console 2026-07-29T08:49:43.835915220Z I0729 08:49:43.835849 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-798649959c-9mnkr 2026-07-29T08:49:43.835915220Z I0729 08:49:43.835876 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-798649959c-9mnkr 2026-07-29T08:49:46.395322109Z I0729 08:49:46.395273 3399 base_network_controller_pods.go:490] [default/openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk] creating logical port openshift-monitoring_openshift-state-metrics-b8f49f5c6-xfvdk for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:49:46.395447704Z I0729 08:49:46.395428 3399 kube.go:234] Patching annotations on pod openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk 2026-07-29T08:49:46.430625837Z I0729 08:49:46.430564 3399 base_network_controller_pods.go:976] [default/openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk] addLogicalPort annotation time took 35.175824ms 2026-07-29T08:49:46.431721508Z I0729 08:49:46.431687 3399 pods.go:273] [openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk] addLogicalPort took 36.434803ms, libovsdb time 810.927µs 2026-07-29T08:49:46.757707404Z I0729 08:49:46.757661 3399 cniserver.go:311] [openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk 8d8b62177c491157c0781988882cbe735aa4dc597a57c87aacd68c99db0fd8ad network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:46.767652193Z I0729 08:49:46.767612 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8f49f5c6-xfvdk, hostIfaceName: 8d8b62177c49115, network: default, NAD default, SandboxID: "8d8b62177c491157c0781988882cbe735aa4dc597a57c87aacd68c99db0fd8ad", PCI device ID: , UID: "5205cb49-9d3c-4ef0-ba43-3331fea2175d", MAC: 0a:58:0a:85:00:10, IPs: [10.133.0.16/23] 2026-07-29T08:49:46.817184701Z I0729 08:49:46.817138 3399 cniserver.go:300] [openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk 8d8b62177c491157c0781988882cbe735aa4dc597a57c87aacd68c99db0fd8ad network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8f49f5c6-xfvdk 8d8b62177c491157c0781988882cbe735aa4dc597a57c87aacd68c99db0fd8ad network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:50:79:53:66:3d\",\"name\":\"8d8b62177c49115\"},{\"mac\":\"0a:58:0a:85:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cea9d531-6802-4609-b405-460b30d5ffff\"}],\"ips\":[{\"address\":\"10.133.0.16/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:48.460487428Z I0729 08:49:48.460445 3399 base_network_controller_pods.go:490] [default/openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj] creating logical port openshift-monitoring_thanos-querier-5cbf85d6fc-mj4jj for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:49:48.460659692Z I0729 08:49:48.460640 3399 kube.go:234] Patching annotations on pod openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj 2026-07-29T08:49:48.477696573Z I0729 08:49:48.477659 3399 base_network_controller_pods.go:976] [default/openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj] addLogicalPort annotation time took 17.064598ms 2026-07-29T08:49:48.479244481Z I0729 08:49:48.479204 3399 pods.go:273] [openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj] addLogicalPort took 18.792591ms, libovsdb time 1.053333ms 2026-07-29T08:49:48.830320459Z I0729 08:49:48.830279 3399 cniserver.go:311] [openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj 931c4fbe2c83c4af70367f77ab5ba58108e37793f5115ee9ff20c14f879752bf network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:48.840339571Z I0729 08:49:48.840293 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5cbf85d6fc-mj4jj, hostIfaceName: 931c4fbe2c83c4a, network: default, NAD default, SandboxID: "931c4fbe2c83c4af70367f77ab5ba58108e37793f5115ee9ff20c14f879752bf", PCI device ID: , UID: "6e674a05-4820-45f2-822d-74dbc96f8d3e", MAC: 0a:58:0a:85:00:11, IPs: [10.133.0.17/23] 2026-07-29T08:49:48.890730565Z I0729 08:49:48.890687 3399 cniserver.go:300] [openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj 931c4fbe2c83c4af70367f77ab5ba58108e37793f5115ee9ff20c14f879752bf network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-5cbf85d6fc-mj4jj 931c4fbe2c83c4af70367f77ab5ba58108e37793f5115ee9ff20c14f879752bf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:a5:9e:bb:47:63\",\"name\":\"931c4fbe2c83c4a\"},{\"mac\":\"0a:58:0a:85:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6a6154ae-11ec-4d6e-b9cf-91fed3ea7656\"}],\"ips\":[{\"address\":\"10.133.0.17/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:51.725635477Z I0729 08:49:51.725595 3399 base_network_controller_pods.go:490] [default/openshift-monitoring/telemeter-client-6948fc4d76-jsx9p] creating logical port openshift-monitoring_telemeter-client-6948fc4d76-jsx9p for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:49:51.725736728Z I0729 08:49:51.725723 3399 kube.go:234] Patching annotations on pod openshift-monitoring/telemeter-client-6948fc4d76-jsx9p 2026-07-29T08:49:51.755195889Z I0729 08:49:51.755157 3399 base_network_controller_pods.go:976] [default/openshift-monitoring/telemeter-client-6948fc4d76-jsx9p] addLogicalPort annotation time took 29.46596ms 2026-07-29T08:49:51.756419631Z I0729 08:49:51.756389 3399 pods.go:273] [openshift-monitoring/telemeter-client-6948fc4d76-jsx9p] addLogicalPort took 30.808626ms, libovsdb time 937.953µs 2026-07-29T08:49:51.921610572Z W0729 08:49:51.921570 3399 util.go:881] Failed to build global endpoints for port TCP/web/9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main 2026-07-29T08:49:51.921610572Z W0729 08:49:51.921591 3399 util.go:881] Failed to build global endpoints for port TCP/tenancy/9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main 2026-07-29T08:49:51.921610572Z W0729 08:49:51.921596 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main 2026-07-29T08:49:52.130609149Z I0729 08:49:52.130563 3399 cniserver.go:311] [openshift-monitoring/telemeter-client-6948fc4d76-jsx9p 7ed1514f42ce0072f6274cd48eef5583b23a0c4d455653572cd0e658f7895323 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:52.143686044Z I0729 08:49:52.143627 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-6948fc4d76-jsx9p, hostIfaceName: 7ed1514f42ce007, network: default, NAD default, SandboxID: "7ed1514f42ce0072f6274cd48eef5583b23a0c4d455653572cd0e658f7895323", PCI device ID: , UID: "6ec93548-a381-4c12-ae5c-a5e6657dc499", MAC: 0a:58:0a:85:00:12, IPs: [10.133.0.18/23] 2026-07-29T08:49:52.203173212Z I0729 08:49:52.203116 3399 cniserver.go:300] [openshift-monitoring/telemeter-client-6948fc4d76-jsx9p 7ed1514f42ce0072f6274cd48eef5583b23a0c4d455653572cd0e658f7895323 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-6948fc4d76-jsx9p 7ed1514f42ce0072f6274cd48eef5583b23a0c4d455653572cd0e658f7895323 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:88:41:62:0e:a5\",\"name\":\"7ed1514f42ce007\"},{\"mac\":\"0a:58:0a:85:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a33f7e90-d5bf-41b2-afba-adf67a01244a\"}],\"ips\":[{\"address\":\"10.133.0.18/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:52.623441047Z I0729 08:49:52.623397 3399 cniserver.go:311] [openshift-multus/network-metrics-daemon-js5hz b6e08b981b0aea3d59693387f45a260ebff11333174eebb59656bcbac1805047 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:52.634472127Z I0729 08:49:52.634428 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-js5hz, hostIfaceName: b6e08b981b0aea3, network: default, NAD default, SandboxID: "b6e08b981b0aea3d59693387f45a260ebff11333174eebb59656bcbac1805047", PCI device ID: , UID: "0f331ed4-9dc7-4481-beea-e463af96775c", MAC: 0a:58:0a:85:00:04, IPs: [10.133.0.4/23] 2026-07-29T08:49:52.686609960Z I0729 08:49:52.686561 3399 cniserver.go:300] [openshift-multus/network-metrics-daemon-js5hz b6e08b981b0aea3d59693387f45a260ebff11333174eebb59656bcbac1805047 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-js5hz b6e08b981b0aea3d59693387f45a260ebff11333174eebb59656bcbac1805047 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:d5:d2:b8:87:9c\",\"name\":\"b6e08b981b0aea3\"},{\"mac\":\"0a:58:0a:85:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a4c2bac-2868-4207-af22-2f69e9049014\"}],\"ips\":[{\"address\":\"10.133.0.4/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:53.317417738Z W0729 08:49:53.317365 3399 util.go:881] Failed to build global endpoints for port TCP/web/9091: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:53.317417738Z W0729 08:49:53.317384 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/9094: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:53.317417738Z W0729 08:49:53.317390 3399 util.go:881] Failed to build global endpoints for port TCP/tenancy/9092: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:53.317417738Z W0729 08:49:53.317394 3399 util.go:881] Failed to build global endpoints for port TCP/tenancy-rules/9093: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:55.932468543Z W0729 08:49:55.932384 3399 util.go:881] Failed to build global endpoints for port TCP/https/10250: empty IP address endpoints for service openshift-monitoring/metrics-server 2026-07-29T08:49:56.346565707Z W0729 08:49:56.346519 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s 2026-07-29T08:49:56.346565707Z W0729 08:49:56.346545 3399 util.go:881] Failed to build global endpoints for port TCP/web/9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s 2026-07-29T08:49:58.227255563Z I0729 08:49:58.227218 3399 base_network_controller_pods.go:490] [default/openshift-console/console-5f5b7fdfff-f29rs] creating logical port openshift-console_console-5f5b7fdfff-f29rs for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:49:58.227368637Z I0729 08:49:58.227335 3399 kube.go:234] Patching annotations on pod openshift-console/console-5f5b7fdfff-f29rs 2026-07-29T08:49:58.250266025Z I0729 08:49:58.250228 3399 base_network_controller_pods.go:976] [default/openshift-console/console-5f5b7fdfff-f29rs] addLogicalPort annotation time took 22.933472ms 2026-07-29T08:49:58.251875367Z I0729 08:49:58.251834 3399 pods.go:273] [openshift-console/console-5f5b7fdfff-f29rs] addLogicalPort took 24.674117ms, libovsdb time 1.097802ms 2026-07-29T08:49:58.344859807Z W0729 08:49:58.344809 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics 2026-07-29T08:49:58.595270723Z I0729 08:49:58.595222 3399 cniserver.go:311] [openshift-console/console-5f5b7fdfff-f29rs 740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:49:58.604632741Z I0729 08:49:58.604589 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-console, podName: console-5f5b7fdfff-f29rs, hostIfaceName: 740214da8c1295f, network: default, NAD default, SandboxID: "740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98", PCI device ID: , UID: "d565af70-d80f-4cf1-9eca-5126f15fb65c", MAC: 0a:58:0a:85:00:13, IPs: [10.133.0.19/23] 2026-07-29T08:49:58.657707416Z I0729 08:49:58.657663 3399 cniserver.go:300] [openshift-console/console-5f5b7fdfff-f29rs 740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5f5b7fdfff-f29rs 740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:78:48:7e:40:7f\",\"name\":\"740214da8c1295f\"},{\"mac\":\"0a:58:0a:85:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b41be324-0460-480a-8f95-b6e7b66a9965\"}],\"ips\":[{\"address\":\"10.133.0.19/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:49:59.302390367Z W0729 08:49:59.302346 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:59.302468699Z W0729 08:49:59.302437 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:59.302468699Z W0729 08:49:59.302452 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:49:59.302468699Z W0729 08:49:59.302459 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier 2026-07-29T08:50:06.383550485Z I0729 08:50:06.383504 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c458b65fd-ddlrg 2026-07-29T08:50:06.383550485Z I0729 08:50:06.383537 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c458b65fd-ddlrg 2026-07-29T08:50:09.020179177Z W0729 08:50:09.020131 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-service-ca-operator/metrics 2026-07-29T08:50:09.394376212Z W0729 08:50:09.394326 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console 2026-07-29T08:50:15.009234195Z I0729 08:50:15.009185 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-68d6dff49c-ckvb5 2026-07-29T08:50:15.009234195Z I0729 08:50:15.009215 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-68d6dff49c-ckvb5 2026-07-29T08:50:21.410339935Z W0729 08:50:21.410296 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-insights/metrics 2026-07-29T08:50:21.410373908Z W0729 08:50:21.410353 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics 2026-07-29T08:50:21.410428863Z W0729 08:50:21.410415 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics 2026-07-29T08:50:22.418278692Z W0729 08:50:22.418233 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics 2026-07-29T08:50:24.070249056Z I0729 08:50:24.070197 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f8bf6b87c-69xgb 2026-07-29T08:50:24.070249056Z I0729 08:50:24.070221 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f8bf6b87c-69xgb 2026-07-29T08:50:35.111577271Z I0729 08:50:35.111529 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b6d85bb57-6q42j 2026-07-29T08:50:35.111577271Z I0729 08:50:35.111565 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b6d85bb57-6q42j 2026-07-29T08:50:44.318098168Z I0729 08:50:44.318050 3399 namespace.go:133] [openshift-kube-storage-version-migrator] updating namespace 2026-07-29T08:51:52.505723824Z I0729 08:51:52.505680 3399 cniserver.go:311] [openshift-console/console-5f5b7fdfff-f29rs 740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:51:52.536770652Z E0729 08:51:52.536729 3399 link_network_manager.go:104] Error while syncing link "740214da8c1295f": failed to get address from link "740214da8c1295f": failed to list addresses for "740214da8c1295f": no such device 2026-07-29T08:51:52.583133065Z I0729 08:51:52.583079 3399 cniserver.go:300] [openshift-console/console-5f5b7fdfff-f29rs 740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5f5b7fdfff-f29rs 740214da8c1295ff7b0122a0242e7b0939e86149d93055b23c87ddfe9c8cad98 network default NAD default NAD key default], result "{}", err 2026-07-29T08:51:52.625954206Z I0729 08:51:52.625908 3399 pods.go:178] Deleting pod: openshift-console/console-5f5b7fdfff-f29rs 2026-07-29T08:51:52.627842836Z I0729 08:51:52.627806 3399 pods.go:238] Attempting to release IPs for pod: openshift-console/console-5f5b7fdfff-f29rs, ips: 10.133.0.19 2026-07-29T08:51:52.754670950Z I0729 08:51:52.754603 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f5b7fdfff-f29rs 2026-07-29T08:51:52.754670950Z I0729 08:51:52.754646 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f5b7fdfff-f29rs 2026-07-29T08:52:09.364219750Z I0729 08:52:09.364168 3399 namespace.go:89] [cert-manager-operator] adding namespace 2026-07-29T08:52:09.364788188Z I0729 08:52:09.364763 3399 namespace.go:93] [cert-manager-operator] adding namespace took 565.18µs 2026-07-29T08:52:09.380322034Z I0729 08:52:09.380283 3399 namespace.go:133] [cert-manager-operator] updating namespace 2026-07-29T08:52:09.390376223Z I0729 08:52:09.390344 3399 namespace.go:133] [cert-manager-operator] updating namespace 2026-07-29T08:52:28.662596823Z I0729 08:52:28.662558 3399 base_network_controller_policy.go:651] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... 2026-07-29T08:52:28.662596823Z I0729 08:52:28.662580 3399 base_network_controller_policy.go:651] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... 2026-07-29T08:52:33.369604760Z I0729 08:52:33.369557 3399 base_network_controller_pods.go:490] [default/cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-55c444bf86-zsdms for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:52:33.369754837Z I0729 08:52:33.369733 3399 kube.go:234] Patching annotations on pod cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms 2026-07-29T08:52:33.389617969Z I0729 08:52:33.389573 3399 base_network_controller_pods.go:976] [default/cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms] addLogicalPort annotation time took 19.879659ms 2026-07-29T08:52:33.390821887Z I0729 08:52:33.390794 3399 pods.go:273] [cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms] addLogicalPort took 21.260942ms, libovsdb time 720.607µs 2026-07-29T08:52:33.727360981Z I0729 08:52:33.727317 3399 cniserver.go:311] [cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms 467f40a96c2728eb47e2e0642c45b07070676c084594cf36ffdfd461d26dcf23 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:52:33.737109289Z I0729 08:52:33.737064 3399 helper_linux.go:533] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-55c444bf86-zsdms, hostIfaceName: 467f40a96c2728e, network: default, NAD default, SandboxID: "467f40a96c2728eb47e2e0642c45b07070676c084594cf36ffdfd461d26dcf23", PCI device ID: , UID: "f50c11d9-9031-4669-a45c-2d7aa52c078c", MAC: 0a:58:0a:85:00:14, IPs: [10.133.0.20/23] 2026-07-29T08:52:33.787119135Z I0729 08:52:33.787072 3399 cniserver.go:300] [cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms 467f40a96c2728eb47e2e0642c45b07070676c084594cf36ffdfd461d26dcf23 network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-55c444bf86-zsdms 467f40a96c2728eb47e2e0642c45b07070676c084594cf36ffdfd461d26dcf23 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:a0:83:7e:ad:ed\",\"name\":\"467f40a96c2728e\"},{\"mac\":\"0a:58:0a:85:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae164a5b-3097-481d-a992-616b44e320d3\"}],\"ips\":[{\"address\":\"10.133.0.20/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:52:37.256564276Z I0729 08:52:37.256509 3399 namespace.go:89] [openshift-lws-operator] adding namespace 2026-07-29T08:52:37.257172323Z I0729 08:52:37.257138 3399 namespace.go:93] [openshift-lws-operator] adding namespace took 605.577µs 2026-07-29T08:52:37.265497712Z I0729 08:52:37.265459 3399 namespace.go:89] [cert-manager] adding namespace 2026-07-29T08:52:37.266016735Z I0729 08:52:37.265991 3399 namespace.go:93] [cert-manager] adding namespace took 506.595µs 2026-07-29T08:52:37.277395427Z I0729 08:52:37.277366 3399 namespace.go:133] [openshift-lws-operator] updating namespace 2026-07-29T08:52:37.298011567Z I0729 08:52:37.297979 3399 namespace.go:133] [cert-manager] updating namespace 2026-07-29T08:52:37.320601742Z I0729 08:52:37.320571 3399 namespace.go:133] [cert-manager] updating namespace 2026-07-29T08:52:37.320666765Z I0729 08:52:37.320615 3399 namespace.go:133] [openshift-lws-operator] updating namespace 2026-07-29T08:52:38.222481655Z I0729 08:52:38.222437 3399 base_network_controller_pods.go:490] [default/openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx] creating logical port openshift-marketplace_77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:52:38.222562623Z I0729 08:52:38.222548 3399 kube.go:234] Patching annotations on pod openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx 2026-07-29T08:52:38.236055029Z I0729 08:52:38.236016 3399 base_network_controller_pods.go:976] [default/openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx] addLogicalPort annotation time took 13.50166ms 2026-07-29T08:52:38.237230859Z I0729 08:52:38.237201 3399 pods.go:273] [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx] addLogicalPort took 14.779575ms, libovsdb time 867.371µs 2026-07-29T08:52:38.581156509Z I0729 08:52:38.581111 3399 cniserver.go:311] [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:52:38.590870074Z I0729 08:52:38.590829 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-marketplace, podName: 77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx, hostIfaceName: a4a269796b58be9, network: default, NAD default, SandboxID: "a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f", PCI device ID: , UID: "f55bc552-568d-4f6b-ad2b-b2a74ff15a6b", MAC: 0a:58:0a:85:00:15, IPs: [10.133.0.21/23] 2026-07-29T08:52:38.642406193Z I0729 08:52:38.642351 3399 cniserver.go:300] [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:6e:cb:09:bf:4d\",\"name\":\"a4a269796b58be9\"},{\"mac\":\"0a:58:0a:85:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d13a3bc4-2b48-4df7-8bcd-5f20d459a8a3\"}],\"ips\":[{\"address\":\"10.133.0.21/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:52:46.619920202Z W0729 08:52:46.619861 3399 util.go:881] Failed to build global endpoints for port TCP/https/10250: empty IP address endpoints for service cert-manager/cert-manager-webhook 2026-07-29T08:52:46.619920202Z W0729 08:52:46.619880 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/9402: empty IP address endpoints for service cert-manager/cert-manager-webhook 2026-07-29T08:52:57.031567370Z I0729 08:52:57.031523 3399 cniserver.go:311] [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:52:57.054632504Z E0729 08:52:57.054597 3399 link_network_manager.go:104] Error while syncing link "a4a269796b58be9": failed to get address from link "a4a269796b58be9": failed to list addresses for "a4a269796b58be9": no such device 2026-07-29T08:52:57.098005551Z I0729 08:52:57.097955 3399 cniserver.go:300] [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx a4a269796b58be948b5d5d175adc80fe3d10f99bfb0b210b9d7a0f2889e67b0f network default NAD default NAD key default], result "{}", err 2026-07-29T08:52:57.137649628Z I0729 08:52:57.137606 3399 pods.go:178] Deleting pod: openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx 2026-07-29T08:52:57.139243673Z I0729 08:52:57.139203 3399 pods.go:238] Attempting to release IPs for pod: openshift-marketplace/77defbb6647882b321b46d16bdefe62633c3425354d3d93571a1d6a87f6qllx, ips: 10.133.0.21 2026-07-29T08:52:57.139271517Z I0729 08:52:57.139247 3399 base_network_controller_policy.go:651] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... 2026-07-29T08:52:57.139451076Z I0729 08:52:57.139431 3399 base_network_controller_policy.go:651] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... 2026-07-29T08:53:01.050926930Z I0729 08:53:01.050875 3399 namespace.go:133] [openshift-lws-operator] updating namespace 2026-07-29T08:53:01.060423450Z I0729 08:53:01.060377 3399 namespace.go:133] [openshift-lws-operator] updating namespace 2026-07-29T08:53:02.826526339Z I0729 08:53:02.826477 3399 base_network_controller_pods.go:490] [default/openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x] creating logical port openshift-lws-operator_openshift-lws-operator-fd8ccff4c-zs26x for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:53:02.826732901Z I0729 08:53:02.826712 3399 kube.go:234] Patching annotations on pod openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x 2026-07-29T08:53:02.836435462Z I0729 08:53:02.836394 3399 base_network_controller_pods.go:976] [default/openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x] addLogicalPort annotation time took 9.720757ms 2026-07-29T08:53:02.837710140Z I0729 08:53:02.837676 3399 pods.go:273] [openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x] addLogicalPort took 11.218297ms, libovsdb time 939.761µs 2026-07-29T08:53:03.262352170Z I0729 08:53:03.262309 3399 cniserver.go:311] [openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x cc3b13a10ecd4491a8eb5f8bd12b091a8810a81d69934f9c3a7e63d9ba3c4840 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:53:03.274107535Z I0729 08:53:03.274046 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-lws-operator, podName: openshift-lws-operator-fd8ccff4c-zs26x, hostIfaceName: cc3b13a10ecd449, network: default, NAD default, SandboxID: "cc3b13a10ecd4491a8eb5f8bd12b091a8810a81d69934f9c3a7e63d9ba3c4840", PCI device ID: , UID: "24542b07-ca74-41bc-a364-bf9801a2cfc0", MAC: 0a:58:0a:85:00:16, IPs: [10.133.0.22/23] 2026-07-29T08:53:03.327190379Z I0729 08:53:03.327141 3399 cniserver.go:300] [openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x cc3b13a10ecd4491a8eb5f8bd12b091a8810a81d69934f9c3a7e63d9ba3c4840 network default NAD default NAD key default] ADD finished CNI request [openshift-lws-operator/openshift-lws-operator-fd8ccff4c-zs26x cc3b13a10ecd4491a8eb5f8bd12b091a8810a81d69934f9c3a7e63d9ba3c4840 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:82:5e:4b:4a:0e\",\"name\":\"cc3b13a10ecd449\"},{\"mac\":\"0a:58:0a:85:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe7c8d10-7fed-41bc-a940-f52d20492f37\"}],\"ips\":[{\"address\":\"10.133.0.22/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:53:06.638877500Z I0729 08:53:06.638834 3399 namespace.go:89] [opendatahub] adding namespace 2026-07-29T08:53:06.639523222Z I0729 08:53:06.639490 3399 namespace.go:93] [opendatahub] adding namespace took 600.543µs 2026-07-29T08:53:06.656698949Z I0729 08:53:06.656663 3399 namespace.go:133] [opendatahub] updating namespace 2026-07-29T08:53:06.668113534Z I0729 08:53:06.668075 3399 namespace.go:133] [opendatahub] updating namespace 2026-07-29T08:53:12.853325836Z I0729 08:53:12.853278 3399 base_network_controller_policy.go:651] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... 2026-07-29T08:53:12.853386276Z I0729 08:53:12.853372 3399 base_network_controller_policy.go:651] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... 2026-07-29T08:53:21.039935094Z I0729 08:53:21.039866 3399 namespace.go:133] [opendatahub] updating namespace 2026-07-29T08:53:22.010134003Z I0729 08:53:22.010058 3399 base_network_controller_pods.go:490] [default/opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck] creating logical port opendatahub_opendatahub-operator-controller-manager-67764bc7cd-758ck for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:53:22.010255574Z I0729 08:53:22.010236 3399 kube.go:234] Patching annotations on pod opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck 2026-07-29T08:53:22.036198672Z I0729 08:53:22.036162 3399 base_network_controller_pods.go:976] [default/opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck] addLogicalPort annotation time took 25.960186ms 2026-07-29T08:53:22.037639843Z I0729 08:53:22.037607 3399 pods.go:273] [opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck] addLogicalPort took 27.565387ms, libovsdb time 1.070635ms 2026-07-29T08:53:22.376657164Z I0729 08:53:22.376615 3399 cniserver.go:311] [opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck 3be094877fb54f42c30faef7ac838e187ede363b285cb803d8fe32f79a165cf8 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:53:22.386957483Z I0729 08:53:22.386913 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: opendatahub-operator-controller-manager-67764bc7cd-758ck, hostIfaceName: 3be094877fb54f4, network: default, NAD default, SandboxID: "3be094877fb54f42c30faef7ac838e187ede363b285cb803d8fe32f79a165cf8", PCI device ID: , UID: "7a8581a4-ccd3-4939-8fd1-69e72cfa7d9c", MAC: 0a:58:0a:85:00:17, IPs: [10.133.0.23/23] 2026-07-29T08:53:22.439077147Z I0729 08:53:22.439029 3399 cniserver.go:300] [opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck 3be094877fb54f42c30faef7ac838e187ede363b285cb803d8fe32f79a165cf8 network default NAD default NAD key default] ADD finished CNI request [opendatahub/opendatahub-operator-controller-manager-67764bc7cd-758ck 3be094877fb54f42c30faef7ac838e187ede363b285cb803d8fe32f79a165cf8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:08:b4:4c:78:7e\",\"name\":\"3be094877fb54f4\"},{\"mac\":\"0a:58:0a:85:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c3d5581-dd92-46d3-9360-296b75bc4085\"}],\"ips\":[{\"address\":\"10.133.0.23/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:53:24.799154027Z W0729 08:53:24.799109 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-lws-operator/lws-controller-manager-metrics-service 2026-07-29T08:53:24.801846325Z W0729 08:53:24.801811 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service openshift-lws-operator/lws-webhook-service 2026-07-29T08:53:26.089563544Z W0729 08:53:26.089518 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service openshift-lws-operator/lws-webhook-service 2026-07-29T08:53:26.090529125Z W0729 08:53:26.090494 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service openshift-lws-operator/lws-controller-manager-metrics-service 2026-07-29T08:53:26.114656166Z W0729 08:53:26.114612 3399 util.go:881] Failed to build global endpoints for port TCP/https/8080: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-metrics-service 2026-07-29T08:53:26.114725007Z W0729 08:53:26.114702 3399 util.go:881] Failed to build global endpoints for port TCP/443/9443: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-service 2026-07-29T08:53:26.170710881Z W0729 08:53:26.170658 3399 util.go:881] Failed to build global endpoints for port TCP/https/8080: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-metrics-service 2026-07-29T08:53:26.170759193Z W0729 08:53:26.170743 3399 util.go:881] Failed to build global endpoints for port TCP/443/9443: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-service 2026-07-29T08:53:26.850390034Z W0729 08:53:26.850334 3399 util.go:881] Failed to build global endpoints for port TCP/https/8080: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-metrics-service 2026-07-29T08:53:26.850437112Z W0729 08:53:26.850427 3399 util.go:881] Failed to build global endpoints for port TCP/443/9443: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-service 2026-07-29T08:53:39.112691856Z W0729 08:53:39.112651 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-metrics-service 2026-07-29T08:53:39.112691856Z W0729 08:53:39.112675 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-metrics-service 2026-07-29T08:53:39.116696958Z W0729 08:53:39.116662 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-service 2026-07-29T08:53:39.116696958Z W0729 08:53:39.116686 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-service 2026-07-29T08:53:39.191932397Z W0729 08:53:39.191869 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-metrics-service 2026-07-29T08:53:39.194527016Z W0729 08:53:39.194433 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/opendatahub-operator-controller-manager-service 2026-07-29T08:53:41.506306623Z I0729 08:53:41.506266 3399 namespace.go:133] [opendatahub] updating namespace 2026-07-29T08:53:41.515789315Z I0729 08:53:41.515752 3399 namespace.go:89] [opendatahub-monitoring] adding namespace 2026-07-29T08:53:41.516401083Z I0729 08:53:41.516370 3399 namespace.go:93] [opendatahub-monitoring] adding namespace took 598.502µs 2026-07-29T08:53:41.532248373Z I0729 08:53:41.532221 3399 namespace.go:133] [opendatahub-monitoring] updating namespace 2026-07-29T08:53:41.547692659Z I0729 08:53:41.547665 3399 namespace.go:133] [opendatahub-monitoring] updating namespace 2026-07-29T08:53:47.029182320Z I0729 08:53:47.029142 3399 base_network_controller_policy.go:1190] Adding network policy opendatahub/opendatahub for network default 2026-07-29T08:53:47.032020316Z I0729 08:53:47.031984 3399 base_network_controller_policy.go:1085] Policy opendatahub/opendatahub added to peer address sets [default-network-controller_LS{ML:{opendatahub.io/generated-namespace: true,},}_LS{}_LNM default-network-controller_LS{ML:{opendatahub.io/application-namespace: true,},}_LS{}_LNM default-network-controller_LS{ML:{network.openshift.io/policy-group: ingress,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-host-network,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-cluster-observability-operator,},}_LS{}_LNM] 2026-07-29T08:53:47.035131892Z I0729 08:53:47.035108 3399 base_network_controller_policy.go:1243] Create network policy opendatahub/opendatahub resources completed, update namespace loglevel 2026-07-29T08:53:47.329001158Z I0729 08:53:47.328955 3399 base_network_controller_pods.go:490] [default/openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg] creating logical port openshift-ingress_kube-auth-proxy-7d544f58d6-7sgdg for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:53:47.329197598Z I0729 08:53:47.329174 3399 kube.go:234] Patching annotations on pod openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg 2026-07-29T08:53:47.345853987Z I0729 08:53:47.345808 3399 base_network_controller_pods.go:976] [default/openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg] addLogicalPort annotation time took 16.686041ms 2026-07-29T08:53:47.347528721Z I0729 08:53:47.347481 3399 pods.go:273] [openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg] addLogicalPort took 18.545167ms, libovsdb time 1.123874ms 2026-07-29T08:53:47.613779655Z I0729 08:53:47.613730 3399 base_network_controller_pods.go:490] [default/opendatahub/odh-model-controller-7998c67dc4-4qdqs] creating logical port opendatahub_odh-model-controller-7998c67dc4-4qdqs for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:53:47.613940078Z I0729 08:53:47.613887 3399 kube.go:234] Patching annotations on pod opendatahub/odh-model-controller-7998c67dc4-4qdqs 2026-07-29T08:53:47.628705057Z I0729 08:53:47.628619 3399 base_network_controller_pods.go:976] [default/opendatahub/odh-model-controller-7998c67dc4-4qdqs] addLogicalPort annotation time took 14.764901ms 2026-07-29T08:53:47.630757084Z I0729 08:53:47.630721 3399 pods.go:273] [opendatahub/odh-model-controller-7998c67dc4-4qdqs] addLogicalPort took 17.01197ms, libovsdb time 1.407823ms 2026-07-29T08:53:48.143840749Z I0729 08:53:48.143796 3399 base_network_controller_pods.go:490] [default/opendatahub/model-serving-api-54d89bd5fc-hjgz9] creating logical port opendatahub_model-serving-api-54d89bd5fc-hjgz9 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:53:48.144015930Z I0729 08:53:48.143993 3399 kube.go:234] Patching annotations on pod opendatahub/model-serving-api-54d89bd5fc-hjgz9 2026-07-29T08:53:48.161437711Z I0729 08:53:48.161397 3399 base_network_controller_pods.go:976] [default/opendatahub/model-serving-api-54d89bd5fc-hjgz9] addLogicalPort annotation time took 17.434603ms 2026-07-29T08:53:48.163098194Z I0729 08:53:48.163065 3399 pods.go:273] [opendatahub/model-serving-api-54d89bd5fc-hjgz9] addLogicalPort took 19.290849ms, libovsdb time 1.168991ms 2026-07-29T08:53:48.294999573Z I0729 08:53:48.294960 3399 cniserver.go:311] [openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg 153a129a33210fd9b9b40187b4a3226fedb82118ca9bb7df7c876da48b482649 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:53:48.307271429Z I0729 08:53:48.307216 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: kube-auth-proxy-7d544f58d6-7sgdg, hostIfaceName: 153a129a33210fd, network: default, NAD default, SandboxID: "153a129a33210fd9b9b40187b4a3226fedb82118ca9bb7df7c876da48b482649", PCI device ID: , UID: "92ff798b-d1cc-4b66-b0bc-17a925537801", MAC: 0a:58:0a:85:00:18, IPs: [10.133.0.24/23] 2026-07-29T08:53:48.359734801Z I0729 08:53:48.359679 3399 cniserver.go:300] [openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg 153a129a33210fd9b9b40187b4a3226fedb82118ca9bb7df7c876da48b482649 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/kube-auth-proxy-7d544f58d6-7sgdg 153a129a33210fd9b9b40187b4a3226fedb82118ca9bb7df7c876da48b482649 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:e1:62:b8:1d:d1\",\"name\":\"153a129a33210fd\"},{\"mac\":\"0a:58:0a:85:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0034c340-1937-42c0-a22c-7769734bb2a0\"}],\"ips\":[{\"address\":\"10.133.0.24/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:53:48.506690554Z I0729 08:53:48.506637 3399 cniserver.go:311] [opendatahub/model-serving-api-54d89bd5fc-hjgz9 106e4a74ae99f9996d5e92b70f09b9201e1085965654bca9d69906046b7e8278 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:53:48.517440956Z I0729 08:53:48.517393 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: model-serving-api-54d89bd5fc-hjgz9, hostIfaceName: 106e4a74ae99f99, network: default, NAD default, SandboxID: "106e4a74ae99f9996d5e92b70f09b9201e1085965654bca9d69906046b7e8278", PCI device ID: , UID: "3d1abe86-75a7-4024-a6ab-db5c3f5b6fe7", MAC: 0a:58:0a:85:00:1a, IPs: [10.133.0.26/23] 2026-07-29T08:53:48.594683513Z I0729 08:53:48.594623 3399 cniserver.go:300] [opendatahub/model-serving-api-54d89bd5fc-hjgz9 106e4a74ae99f9996d5e92b70f09b9201e1085965654bca9d69906046b7e8278 network default NAD default NAD key default] ADD finished CNI request [opendatahub/model-serving-api-54d89bd5fc-hjgz9 106e4a74ae99f9996d5e92b70f09b9201e1085965654bca9d69906046b7e8278 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:11:94:5a:78:db\",\"name\":\"106e4a74ae99f99\"},{\"mac\":\"0a:58:0a:85:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/23c53c46-0e44-43ea-907f-8160dfb5c6e9\"}],\"ips\":[{\"address\":\"10.133.0.26/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:53:48.608344946Z I0729 08:53:48.608306 3399 cniserver.go:311] [opendatahub/odh-model-controller-7998c67dc4-4qdqs 6196e8f1c85a86df1b6d467aa55beaf951f0140e3654a3aeaf4495763a01efea network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:53:48.621234814Z I0729 08:53:48.621186 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: odh-model-controller-7998c67dc4-4qdqs, hostIfaceName: 6196e8f1c85a86d, network: default, NAD default, SandboxID: "6196e8f1c85a86df1b6d467aa55beaf951f0140e3654a3aeaf4495763a01efea", PCI device ID: , UID: "02fc77ac-50ff-4ab4-9b01-501c1770d014", MAC: 0a:58:0a:85:00:19, IPs: [10.133.0.25/23] 2026-07-29T08:53:48.677229801Z I0729 08:53:48.677176 3399 cniserver.go:300] [opendatahub/odh-model-controller-7998c67dc4-4qdqs 6196e8f1c85a86df1b6d467aa55beaf951f0140e3654a3aeaf4495763a01efea network default NAD default NAD key default] ADD finished CNI request [opendatahub/odh-model-controller-7998c67dc4-4qdqs 6196e8f1c85a86df1b6d467aa55beaf951f0140e3654a3aeaf4495763a01efea network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:06:2f:bc:9b:67\",\"name\":\"6196e8f1c85a86d\"},{\"mac\":\"0a:58:0a:85:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6d9a48a9-4df2-4bcd-aff8-1b2c4412bf34\"}],\"ips\":[{\"address\":\"10.133.0.25/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:53:48.872386682Z I0729 08:53:48.872341 3399 base_network_controller_pods.go:490] [default/openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4] creating logical port openshift-ingress_istiod-openshift-gateway-7b7995d5cd-wpdr4 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:53:48.872587674Z I0729 08:53:48.872560 3399 kube.go:234] Patching annotations on pod openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 2026-07-29T08:53:48.899654719Z I0729 08:53:48.899614 3399 base_network_controller_pods.go:976] [default/openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4] addLogicalPort annotation time took 27.102412ms 2026-07-29T08:53:48.901179325Z I0729 08:53:48.901132 3399 pods.go:273] [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4] addLogicalPort took 28.809752ms, libovsdb time 1.029701ms 2026-07-29T08:53:49.014309895Z I0729 08:53:49.014271 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=6 2026-07-29T08:53:49.245464773Z I0729 08:53:49.245420 3399 cniserver.go:311] [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:53:49.257637895Z I0729 08:53:49.257584 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: istiod-openshift-gateway-7b7995d5cd-wpdr4, hostIfaceName: ed3080ffce4f266, network: default, NAD default, SandboxID: "ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98", PCI device ID: , UID: "18a1ed3b-46f4-44e8-8188-af458a096599", MAC: 0a:58:0a:85:00:1b, IPs: [10.133.0.27/23] 2026-07-29T08:53:49.325675785Z I0729 08:53:49.325620 3399 cniserver.go:300] [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:b6:97:e7:d4:43\",\"name\":\"ed3080ffce4f266\"},{\"mac\":\"0a:58:0a:85:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b88b978-fdbe-4c73-b1a7-6b3e3b55e42f\"}],\"ips\":[{\"address\":\"10.133.0.27/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:53:50.018549879Z I0729 08:53:50.018501 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/kube-auth-proxy for network default 2026-07-29T08:53:50.020261043Z I0729 08:53:50.020227 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/kube-auth-proxy added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.networking.k8s.io/gateway-name: data-science-gateway,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T08:53:50.024233723Z I0729 08:53:50.024209 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/kube-auth-proxy resources completed, update namespace loglevel 2026-07-29T08:53:53.999219107Z I0729 08:53:53.999174 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 2026-07-29T08:53:54.346381929Z W0729 08:53:54.346340 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service opendatahub/model-serving-api 2026-07-29T08:53:54.346438756Z W0729 08:53:54.346427 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service opendatahub/model-serving-api 2026-07-29T08:53:54.366589614Z W0729 08:53:54.366547 3399 util.go:881] Failed to build global endpoints for port TCP/https-dns/15012: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:54.366649413Z W0729 08:53:54.366638 3399 util.go:881] Failed to build global endpoints for port TCP/https-webhook/15017: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:54.366678100Z W0729 08:53:54.366669 3399 util.go:881] Failed to build global endpoints for port TCP/grpc-xds/15010: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:54.366702787Z W0729 08:53:54.366694 3399 util.go:881] Failed to build global endpoints for port TCP/http-monitoring/15014: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:54.381755730Z W0729 08:53:54.381714 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service opendatahub/odh-model-controller-webhook-service 2026-07-29T08:53:54.384398306Z W0729 08:53:54.384359 3399 util.go:881] Failed to build global endpoints for port TCP/http/8080: empty IP address endpoints for service opendatahub/odh-model-controller-metrics-service 2026-07-29T08:53:58.365670671Z W0729 08:53:58.365609 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15012: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:58.365670671Z W0729 08:53:58.365640 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15017: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:58.365670671Z W0729 08:53:58.365650 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15010: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:58.365670671Z W0729 08:53:58.365657 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15014: empty IP address endpoints for service openshift-ingress/istiod-openshift-gateway 2026-07-29T08:53:59.295097325Z W0729 08:53:59.295052 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/data-science-gateway-data-science-gateway-class 2026-07-29T08:53:59.295097325Z W0729 08:53:59.295070 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/data-science-gateway-data-science-gateway-class 2026-07-29T08:54:00.007501708Z I0729 08:54:00.007463 3399 base_network_controller_pods.go:490] [default/opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr] creating logical port opendatahub_kserve-controller-manager-d6f4b65b8-r9fsr for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:00.007686885Z I0729 08:54:00.007665 3399 kube.go:234] Patching annotations on pod opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr 2026-07-29T08:54:00.040921470Z I0729 08:54:00.040873 3399 base_network_controller_pods.go:976] [default/opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr] addLogicalPort annotation time took 33.241977ms 2026-07-29T08:54:00.042267237Z I0729 08:54:00.042235 3399 pods.go:273] [opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr] addLogicalPort took 34.792121ms, libovsdb time 970.051µs 2026-07-29T08:54:00.057999881Z I0729 08:54:00.057966 3399 base_network_controller_pods.go:490] [default/opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r] creating logical port opendatahub_llmisvc-controller-manager-777c6bd7d5-s8t5r for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:00.058150744Z I0729 08:54:00.058128 3399 kube.go:234] Patching annotations on pod opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r 2026-07-29T08:54:00.076883138Z I0729 08:54:00.076848 3399 base_network_controller_policy.go:1190] Adding network policy opendatahub/kserve-controller-manager for network default 2026-07-29T08:54:00.076936578Z I0729 08:54:00.076922 3399 base_network_controller_policy.go:1085] Policy opendatahub/kserve-controller-manager added to peer address sets [] 2026-07-29T08:54:00.082658146Z I0729 08:54:00.082623 3399 base_network_controller_pods.go:976] [default/opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r] addLogicalPort annotation time took 24.540765ms 2026-07-29T08:54:00.083826440Z I0729 08:54:00.083798 3399 pods.go:273] [opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r] addLogicalPort took 25.84971ms, libovsdb time 634.408µs 2026-07-29T08:54:00.084727990Z I0729 08:54:00.084707 3399 base_network_controller_policy.go:1243] Create network policy opendatahub/kserve-controller-manager resources completed, update namespace loglevel 2026-07-29T08:54:00.374579630Z I0729 08:54:00.374538 3399 cniserver.go:311] [opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr 3b409aefd426ba71e6e95f8c67ea58acf3cccb73715db30523d060d6f6463e7f network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:00.383946379Z I0729 08:54:00.383903 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: kserve-controller-manager-d6f4b65b8-r9fsr, hostIfaceName: 3b409aefd426ba7, network: default, NAD default, SandboxID: "3b409aefd426ba71e6e95f8c67ea58acf3cccb73715db30523d060d6f6463e7f", PCI device ID: , UID: "b47d770a-97bc-40e5-b878-bc2fcb787077", MAC: 0a:58:0a:85:00:1c, IPs: [10.133.0.28/23] 2026-07-29T08:54:00.436928558Z I0729 08:54:00.436871 3399 cniserver.go:300] [opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr 3b409aefd426ba71e6e95f8c67ea58acf3cccb73715db30523d060d6f6463e7f network default NAD default NAD key default] ADD finished CNI request [opendatahub/kserve-controller-manager-d6f4b65b8-r9fsr 3b409aefd426ba71e6e95f8c67ea58acf3cccb73715db30523d060d6f6463e7f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:c3:7e:b7:3b:af\",\"name\":\"3b409aefd426ba7\"},{\"mac\":\"0a:58:0a:85:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fd7482f4-717e-492b-8ede-58fa34f5600c\"}],\"ips\":[{\"address\":\"10.133.0.28/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:54:01.021141712Z I0729 08:54:01.021090 3399 cniserver.go:311] [opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r 912e9f311f6e59010b61445e6573505b735274823d8e9d5d29aae7ee6c3921a5 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:01.031215861Z I0729 08:54:01.031168 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: llmisvc-controller-manager-777c6bd7d5-s8t5r, hostIfaceName: 912e9f311f6e590, network: default, NAD default, SandboxID: "912e9f311f6e59010b61445e6573505b735274823d8e9d5d29aae7ee6c3921a5", PCI device ID: , UID: "43eb1f99-cf03-4f6b-8bdd-9dd771a302f4", MAC: 0a:58:0a:85:00:1d, IPs: [10.133.0.29/23] 2026-07-29T08:54:01.084267228Z I0729 08:54:01.084214 3399 cniserver.go:300] [opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r 912e9f311f6e59010b61445e6573505b735274823d8e9d5d29aae7ee6c3921a5 network default NAD default NAD key default] ADD finished CNI request [opendatahub/llmisvc-controller-manager-777c6bd7d5-s8t5r 912e9f311f6e59010b61445e6573505b735274823d8e9d5d29aae7ee6c3921a5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:ef:78:85:68:65\",\"name\":\"912e9f311f6e590\"},{\"mac\":\"0a:58:0a:85:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/519635df-ea60-4d0e-80a8-1e64032e72ae\"}],\"ips\":[{\"address\":\"10.133.0.29/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:54:03.001048561Z I0729 08:54:03.001001 3399 cniserver.go:311] [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:54:03.034505313Z E0729 08:54:03.034470 3399 link_network_manager.go:104] Error while syncing link "ed3080ffce4f266": failed to get address from link "ed3080ffce4f266": failed to list addresses for "ed3080ffce4f266": no such device 2026-07-29T08:54:03.075312250Z I0729 08:54:03.075267 3399 cniserver.go:300] [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 ed3080ffce4f2667f8db53ff2c1746a75e15e8f823f3cfa756d8ee019f191e98 network default NAD default NAD key default], result "{}", err 2026-07-29T08:54:03.159969702Z I0729 08:54:03.159930 3399 pods.go:178] Deleting pod: openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 2026-07-29T08:54:03.161749111Z I0729 08:54:03.161713 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4, ips: 10.133.0.27 2026-07-29T08:54:03.396001144Z W0729 08:54:03.395955 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service opendatahub/kserve-webhook-server-service 2026-07-29T08:54:03.405584153Z I0729 08:54:03.405534 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 2026-07-29T08:54:03.405680606Z I0729 08:54:03.405665 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/istiod-openshift-gateway-7b7995d5cd-wpdr4 2026-07-29T08:54:04.426783505Z W0729 08:54:04.426736 3399 util.go:881] Failed to build global endpoints for port TCP/https/9443: empty IP address endpoints for service opendatahub/llmisvc-webhook-server-service 2026-07-29T08:54:04.428234209Z W0729 08:54:04.428211 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service opendatahub/llmisvc-controller-manager-service 2026-07-29T08:54:05.394041506Z W0729 08:54:05.393995 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/odh-model-controller-webhook-service 2026-07-29T08:54:05.398403848Z W0729 08:54:05.398367 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service opendatahub/odh-model-controller-metrics-service 2026-07-29T08:54:05.425277241Z W0729 08:54:05.425242 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service opendatahub/model-serving-api 2026-07-29T08:54:05.425277241Z W0729 08:54:05.425264 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service opendatahub/model-serving-api 2026-07-29T08:54:21.898408912Z I0729 08:54:21.898372 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=150 2026-07-29T08:54:34.401999402Z W0729 08:54:34.401955 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/kserve-webhook-server-service 2026-07-29T08:54:35.405697826Z W0729 08:54:35.405647 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/llmisvc-webhook-server-service 2026-07-29T08:54:35.405954694Z W0729 08:54:35.405920 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service opendatahub/llmisvc-controller-manager-service 2026-07-29T08:54:42.986104656Z I0729 08:54:42.986058 3399 namespace.go:89] [kuadrant-system] adding namespace 2026-07-29T08:54:42.986802099Z I0729 08:54:42.986765 3399 namespace.go:93] [kuadrant-system] adding namespace took 628.413µs 2026-07-29T08:54:43.004926085Z I0729 08:54:43.004879 3399 namespace.go:133] [kuadrant-system] updating namespace 2026-07-29T08:54:43.011385276Z I0729 08:54:43.011350 3399 namespace.go:133] [kuadrant-system] updating namespace 2026-07-29T08:54:43.146397394Z I0729 08:54:43.146349 3399 base_network_controller_policy.go:1190] Adding network policy kuadrant-system/kuadrant-operator-catalog-grpc-server for network default 2026-07-29T08:54:43.146475689Z I0729 08:54:43.146456 3399 base_network_controller_policy.go:1085] Policy kuadrant-system/kuadrant-operator-catalog-grpc-server added to peer address sets [] 2026-07-29T08:54:43.148649924Z I0729 08:54:43.148618 3399 base_network_controller_policy.go:1243] Create network policy kuadrant-system/kuadrant-operator-catalog-grpc-server resources completed, update namespace loglevel 2026-07-29T08:54:43.150216664Z I0729 08:54:43.150186 3399 base_network_controller_policy.go:1190] Adding network policy kuadrant-system/kuadrant-operator-catalog-unpack-bundles for network default 2026-07-29T08:54:43.150216664Z I0729 08:54:43.150209 3399 base_network_controller_policy.go:1085] Policy kuadrant-system/kuadrant-operator-catalog-unpack-bundles added to peer address sets [] 2026-07-29T08:54:43.151354915Z I0729 08:54:43.151331 3399 base_network_controller_policy.go:1243] Create network policy kuadrant-system/kuadrant-operator-catalog-unpack-bundles resources completed, update namespace loglevel 2026-07-29T08:54:43.186366522Z I0729 08:54:43.186324 3399 base_network_controller_pods.go:490] [default/kuadrant-system/kuadrant-operator-catalog-cqvpv] creating logical port kuadrant-system_kuadrant-operator-catalog-cqvpv for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:43.186456770Z I0729 08:54:43.186445 3399 kube.go:234] Patching annotations on pod kuadrant-system/kuadrant-operator-catalog-cqvpv 2026-07-29T08:54:43.200170055Z I0729 08:54:43.200128 3399 base_network_controller_pods.go:976] [default/kuadrant-system/kuadrant-operator-catalog-cqvpv] addLogicalPort annotation time took 13.721122ms 2026-07-29T08:54:43.201624815Z I0729 08:54:43.201587 3399 pods.go:273] [kuadrant-system/kuadrant-operator-catalog-cqvpv] addLogicalPort took 15.280971ms, libovsdb time 915.984µs 2026-07-29T08:54:43.552088673Z I0729 08:54:43.552043 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-catalog-cqvpv a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:43.561572082Z I0729 08:54:43.561528 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: kuadrant-operator-catalog-cqvpv, hostIfaceName: a0cd1abf05468fb, network: default, NAD default, SandboxID: "a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd", PCI device ID: , UID: "999e1bd3-4993-4e4e-ba82-49e920693b9b", MAC: 0a:58:0a:85:00:1e, IPs: [10.133.0.30/23] 2026-07-29T08:54:43.616600923Z I0729 08:54:43.616549 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-catalog-cqvpv a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/kuadrant-operator-catalog-cqvpv a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:e3:9a:ed:15:a5\",\"name\":\"a0cd1abf05468fb\"},{\"mac\":\"0a:58:0a:85:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a752288-ae1d-4115-a99b-e3b01bc096c8\"}],\"ips\":[{\"address\":\"10.133.0.30/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:54:43.754761884Z I0729 08:54:43.754716 3399 base_network_controller_pods.go:490] [default/kuadrant-system/kuadrant-operator-catalog-cr8qp] creating logical port kuadrant-system_kuadrant-operator-catalog-cr8qp for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:43.754931431Z I0729 08:54:43.754886 3399 kube.go:234] Patching annotations on pod kuadrant-system/kuadrant-operator-catalog-cr8qp 2026-07-29T08:54:43.764678896Z I0729 08:54:43.764629 3399 base_network_controller_pods.go:976] [default/kuadrant-system/kuadrant-operator-catalog-cr8qp] addLogicalPort annotation time took 9.792982ms 2026-07-29T08:54:43.766040612Z I0729 08:54:43.766004 3399 pods.go:273] [kuadrant-system/kuadrant-operator-catalog-cr8qp] addLogicalPort took 11.310652ms, libovsdb time 864.749µs 2026-07-29T08:54:44.116364286Z I0729 08:54:44.116319 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-catalog-cr8qp 1c318f5d0ad6c4601c85a436b162b5db51a3071ab59df17858b4bbba427639bd network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:44.125808930Z I0729 08:54:44.125763 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: kuadrant-operator-catalog-cr8qp, hostIfaceName: 1c318f5d0ad6c46, network: default, NAD default, SandboxID: "1c318f5d0ad6c4601c85a436b162b5db51a3071ab59df17858b4bbba427639bd", PCI device ID: , UID: "9277a86e-6ace-49d0-a241-c33f168367f2", MAC: 0a:58:0a:85:00:1f, IPs: [10.133.0.31/23] 2026-07-29T08:54:44.188464613Z I0729 08:54:44.188420 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-catalog-cr8qp 1c318f5d0ad6c4601c85a436b162b5db51a3071ab59df17858b4bbba427639bd network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/kuadrant-operator-catalog-cr8qp 1c318f5d0ad6c4601c85a436b162b5db51a3071ab59df17858b4bbba427639bd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:68:88:71:3c:a4\",\"name\":\"1c318f5d0ad6c46\"},{\"mac\":\"0a:58:0a:85:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20105242-188f-4939-adf1-34e9817004fd\"}],\"ips\":[{\"address\":\"10.133.0.31/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:54:46.595599939Z W0729 08:54:46.595548 3399 util.go:881] Failed to build global endpoints for port TCP/grpc/50051: empty IP address endpoints for service kuadrant-system/kuadrant-operator-catalog 2026-07-29T08:54:46.613625161Z W0729 08:54:46.613589 3399 util.go:881] Failed to build global endpoints for port TCP/grpc/50051: empty IP address endpoints for service kuadrant-system/kuadrant-operator-catalog 2026-07-29T08:54:46.730138105Z I0729 08:54:46.730097 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-catalog-cqvpv a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:54:46.753772597Z E0729 08:54:46.753734 3399 link_network_manager.go:104] Error while syncing link "a0cd1abf05468fb": failed to get address from link "a0cd1abf05468fb": failed to list addresses for "a0cd1abf05468fb": no such device 2026-07-29T08:54:46.796786570Z I0729 08:54:46.796732 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-catalog-cqvpv a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/kuadrant-operator-catalog-cqvpv a0cd1abf05468fb22e6637d940bcc02ff534bdaf16c81d8633e64a84a63c98cd network default NAD default NAD key default], result "{}", err 2026-07-29T08:54:46.835754541Z I0729 08:54:46.835710 3399 pods.go:178] Deleting pod: kuadrant-system/kuadrant-operator-catalog-cqvpv 2026-07-29T08:54:46.837121584Z I0729 08:54:46.837093 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/kuadrant-operator-catalog-cqvpv, ips: 10.133.0.30 2026-07-29T08:54:46.837159391Z I0729 08:54:46.837145 3399 base_network_controller_policy.go:651] Processing NetworkPolicy kuadrant-system/kuadrant-operator-catalog-grpc-server to delete 1 local pods... 2026-07-29T08:54:46.840583659Z W0729 08:54:46.840557 3399 util.go:881] Failed to build global endpoints for port TCP/grpc/50051: empty IP address endpoints for service kuadrant-system/kuadrant-operator-catalog 2026-07-29T08:54:47.605191598Z I0729 08:54:47.605153 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector kuadrant-system/kuadrant-operator-catalog-cqvpv 2026-07-29T08:54:47.605191598Z I0729 08:54:47.605186 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/kuadrant-operator-catalog-cqvpv 2026-07-29T08:54:47.605213216Z I0729 08:54:47.605205 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/kuadrant-operator-catalog-cqvpv 2026-07-29T08:54:54.668066868Z W0729 08:54:54.668026 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 50051: empty IP address endpoints for service kuadrant-system/kuadrant-operator-catalog 2026-07-29T08:54:58.790674587Z I0729 08:54:58.790627 3399 base_network_controller_pods.go:490] [default/kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx] creating logical port kuadrant-system_19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:58.790765417Z I0729 08:54:58.790746 3399 kube.go:234] Patching annotations on pod kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 2026-07-29T08:54:58.815494275Z I0729 08:54:58.815451 3399 base_network_controller_pods.go:976] [default/kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx] addLogicalPort annotation time took 24.74456ms 2026-07-29T08:54:58.816749488Z I0729 08:54:58.816718 3399 pods.go:273] [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx] addLogicalPort took 26.110851ms, libovsdb time 869.074µs 2026-07-29T08:54:59.152065197Z I0729 08:54:59.152018 3399 cniserver.go:311] [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:59.163178396Z I0729 08:54:59.163120 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: 19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx, hostIfaceName: 42516e46b64d9b9, network: default, NAD default, SandboxID: "42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a", PCI device ID: , UID: "1d73bbc1-fedc-4bc9-bf95-c5c8d6d13b6b", MAC: 0a:58:0a:85:00:20, IPs: [10.133.0.32/23] 2026-07-29T08:54:59.203766792Z I0729 08:54:59.203690 3399 base_network_controller_pods.go:490] [default/kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt] creating logical port kuadrant-system_0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:59.203877801Z I0729 08:54:59.203854 3399 kube.go:234] Patching annotations on pod kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 2026-07-29T08:54:59.215929335Z I0729 08:54:59.215863 3399 base_network_controller_pods.go:976] [default/kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt] addLogicalPort annotation time took 12.05069ms 2026-07-29T08:54:59.216794413Z I0729 08:54:59.216747 3399 cniserver.go:300] [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:b1:f0:3b:2f:57\",\"name\":\"42516e46b64d9b9\"},{\"mac\":\"0a:58:0a:85:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8dcf3ef-d4d1-41a8-99c0-d4fa1be291f4\"}],\"ips\":[{\"address\":\"10.133.0.32/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:54:59.217732015Z I0729 08:54:59.217696 3399 pods.go:273] [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt] addLogicalPort took 14.028872ms, libovsdb time 902.596µs 2026-07-29T08:54:59.567695260Z I0729 08:54:59.567648 3399 cniserver.go:311] [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:59.578878348Z I0729 08:54:59.578832 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: 0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt, hostIfaceName: 5260fad7b0ceb6d, network: default, NAD default, SandboxID: "5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a", PCI device ID: , UID: "ab1dd62a-eea5-4cbe-9a6a-dad207e0b559", MAC: 0a:58:0a:85:00:21, IPs: [10.133.0.33/23] 2026-07-29T08:54:59.596434608Z I0729 08:54:59.596400 3399 base_network_controller_pods.go:490] [default/kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m] creating logical port kuadrant-system_5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:54:59.596577729Z I0729 08:54:59.596563 3399 kube.go:234] Patching annotations on pod kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 2026-07-29T08:54:59.607451157Z I0729 08:54:59.607412 3399 base_network_controller_pods.go:976] [default/kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m] addLogicalPort annotation time took 10.874884ms 2026-07-29T08:54:59.609089086Z I0729 08:54:59.609060 3399 pods.go:273] [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m] addLogicalPort took 12.67269ms, libovsdb time 894.89µs 2026-07-29T08:54:59.635603234Z I0729 08:54:59.635536 3399 cniserver.go:300] [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:e4:b7:82:9b:e8\",\"name\":\"5260fad7b0ceb6d\"},{\"mac\":\"0a:58:0a:85:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c91d7f10-0a4a-4204-b045-6713ba3bdbff\"}],\"ips\":[{\"address\":\"10.133.0.33/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:54:59.960691115Z I0729 08:54:59.960649 3399 cniserver.go:311] [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:54:59.972230145Z I0729 08:54:59.972190 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: 5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m, hostIfaceName: 28aac10a8262648, network: default, NAD default, SandboxID: "28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194", PCI device ID: , UID: "b7e04cf2-881f-42dc-b632-0d4f8313b704", MAC: 0a:58:0a:85:00:22, IPs: [10.133.0.34/23] 2026-07-29T08:55:00.017164160Z I0729 08:55:00.017120 3399 base_network_controller_pods.go:490] [default/kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw] creating logical port kuadrant-system_9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:00.017420200Z I0729 08:55:00.017384 3399 kube.go:234] Patching annotations on pod kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw 2026-07-29T08:55:00.030737012Z E0729 08:55:00.030700 3399 kube.go:244] Error in patching annotations on pod kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw: the server rejected our request due to an error in our request 2026-07-29T08:55:00.041408684Z I0729 08:55:00.041379 3399 kube.go:234] Patching annotations on pod kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw 2026-07-29T08:55:00.047935790Z I0729 08:55:00.047867 3399 cniserver.go:300] [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194 network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:d8:1b:bd:3f:82\",\"name\":\"28aac10a8262648\"},{\"mac\":\"0a:58:0a:85:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f1f9f6b6-f770-44c4-bf09-c32bfa83d231\"}],\"ips\":[{\"address\":\"10.133.0.34/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:00.061134355Z I0729 08:55:00.061084 3399 base_network_controller_pods.go:976] [default/kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw] addLogicalPort annotation time took 43.762994ms 2026-07-29T08:55:00.062353890Z I0729 08:55:00.062312 3399 pods.go:273] [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw] addLogicalPort took 45.221324ms, libovsdb time 860.228µs 2026-07-29T08:55:00.383098621Z I0729 08:55:00.383049 3399 cniserver.go:311] [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:00.392958013Z I0729 08:55:00.392906 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: 9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw, hostIfaceName: a8d7be043fa6701, network: default, NAD default, SandboxID: "a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d", PCI device ID: , UID: "1cb3b2dd-187f-419d-b933-bfaf616e2875", MAC: 0a:58:0a:85:00:23, IPs: [10.133.0.35/23] 2026-07-29T08:55:00.449525618Z I0729 08:55:00.449477 3399 cniserver.go:300] [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:dd:0d:2b:24:b9\",\"name\":\"a8d7be043fa6701\"},{\"mac\":\"0a:58:0a:85:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/991e8342-c60a-4207-8af5-03d6318befa6\"}],\"ips\":[{\"address\":\"10.133.0.35/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:01.339137592Z I0729 08:55:01.339099 3399 base_network_controller_pods.go:490] [default/openshift-console/console-65db55cc87-b7thh] creating logical port openshift-console_console-65db55cc87-b7thh for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:01.339295503Z I0729 08:55:01.339277 3399 kube.go:234] Patching annotations on pod openshift-console/console-65db55cc87-b7thh 2026-07-29T08:55:01.362273752Z I0729 08:55:01.362230 3399 base_network_controller_pods.go:976] [default/openshift-console/console-65db55cc87-b7thh] addLogicalPort annotation time took 22.994765ms 2026-07-29T08:55:01.363880862Z I0729 08:55:01.363849 3399 pods.go:273] [openshift-console/console-65db55cc87-b7thh] addLogicalPort took 24.777679ms, libovsdb time 997.901µs 2026-07-29T08:55:01.712276970Z I0729 08:55:01.712234 3399 cniserver.go:311] [openshift-console/console-65db55cc87-b7thh 6689b82b364dde2bdbf8c982d7ffbd695eb5b115feebb1204509fb0d61b27a10 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:01.722692584Z I0729 08:55:01.722634 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-console, podName: console-65db55cc87-b7thh, hostIfaceName: 6689b82b364dde2, network: default, NAD default, SandboxID: "6689b82b364dde2bdbf8c982d7ffbd695eb5b115feebb1204509fb0d61b27a10", PCI device ID: , UID: "3d4af3b4-68d4-444f-8a0a-c34935696b23", MAC: 0a:58:0a:85:00:24, IPs: [10.133.0.36/23] 2026-07-29T08:55:01.790456236Z I0729 08:55:01.790391 3399 cniserver.go:300] [openshift-console/console-65db55cc87-b7thh 6689b82b364dde2bdbf8c982d7ffbd695eb5b115feebb1204509fb0d61b27a10 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-65db55cc87-b7thh 6689b82b364dde2bdbf8c982d7ffbd695eb5b115feebb1204509fb0d61b27a10 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:bf:db:b2:c8:b2\",\"name\":\"6689b82b364dde2\"},{\"mac\":\"0a:58:0a:85:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0dadc5de-e54f-4042-9723-e7f9f6e85c81\"}],\"ips\":[{\"address\":\"10.133.0.36/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:02.716282738Z I0729 08:55:02.716235 3399 cniserver.go:311] [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:02.745078138Z E0729 08:55:02.745041 3399 link_network_manager.go:104] Error while syncing link "42516e46b64d9b9": failed to get address from link "42516e46b64d9b9": failed to list addresses for "42516e46b64d9b9": no such device 2026-07-29T08:55:02.795373570Z I0729 08:55:02.795324 3399 cniserver.go:300] [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 42516e46b64d9b9a459457a3adda1a27af5242b15f08462f86d8e98be08eb59a network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:02.831721582Z I0729 08:55:02.831671 3399 pods.go:178] Deleting pod: kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx 2026-07-29T08:55:02.833767209Z I0729 08:55:02.833735 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/19cb86e64775c5699d5aacf881a09c2d51e7f55f9e1ff096f2a667c5e0t5nwx, ips: 10.133.0.32 2026-07-29T08:55:02.833804809Z I0729 08:55:02.833787 3399 base_network_controller_policy.go:651] Processing NetworkPolicy kuadrant-system/kuadrant-operator-catalog-unpack-bundles to delete 1 local pods... 2026-07-29T08:55:03.725102694Z I0729 08:55:03.725057 3399 cniserver.go:311] [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:03.743720837Z E0729 08:55:03.743680 3399 link_network_manager.go:104] Error while syncing link "28aac10a8262648": failed to get address from link "28aac10a8262648": failed to list addresses for "28aac10a8262648": no such device 2026-07-29T08:55:03.745391113Z E0729 08:55:03.745361 3399 link_network_manager.go:104] Error while syncing link "28aac10a8262648": failed to get address from link "28aac10a8262648": failed to list addresses for "28aac10a8262648": no such device 2026-07-29T08:55:03.802727302Z I0729 08:55:03.802673 3399 cniserver.go:300] [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194 network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 28aac10a8262648736da3cfff0fb4a16754dc7565d3548cee31d424c2e455194 network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:03.845537442Z I0729 08:55:03.845492 3399 pods.go:178] Deleting pod: kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m 2026-07-29T08:55:03.847300992Z I0729 08:55:03.847264 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/5fc36eb1065777914bfe0ff85f9a202e2a3bafefb563d9e2994474ed73rg42m, ips: 10.133.0.34 2026-07-29T08:55:03.847327964Z I0729 08:55:03.847317 3399 base_network_controller_policy.go:651] Processing NetworkPolicy kuadrant-system/kuadrant-operator-catalog-unpack-bundles to delete 1 local pods... 2026-07-29T08:55:04.731420590Z I0729 08:55:04.731376 3399 cniserver.go:311] [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:04.731420590Z I0729 08:55:04.731400 3399 cniserver.go:311] [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:04.750353642Z E0729 08:55:04.750318 3399 link_network_manager.go:104] Error while syncing link "5260fad7b0ceb6d": failed to get address from link "5260fad7b0ceb6d": failed to list addresses for "5260fad7b0ceb6d": no such device 2026-07-29T08:55:04.792485904Z E0729 08:55:04.792451 3399 link_network_manager.go:104] Error while syncing link "a8d7be043fa6701": failed to get address from link "a8d7be043fa6701": failed to list addresses for "a8d7be043fa6701": no such device 2026-07-29T08:55:04.798266954Z I0729 08:55:04.798230 3399 cniserver.go:300] [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 5260fad7b0ceb6dcdbbb4b783cb555767cfc881b67a0cdfe8ba224007cb4ae1a network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:04.834493057Z I0729 08:55:04.834442 3399 cniserver.go:300] [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw a8d7be043fa6701c9de0772992e98842116490ff01096bdd85957f7afce5594d network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:04.854178078Z I0729 08:55:04.854136 3399 pods.go:178] Deleting pod: kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 2026-07-29T08:55:04.856128111Z I0729 08:55:04.856094 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt, ips: 10.133.0.33 2026-07-29T08:55:04.856181858Z I0729 08:55:04.856166 3399 base_network_controller_policy.go:651] Processing NetworkPolicy kuadrant-system/kuadrant-operator-catalog-unpack-bundles to delete 1 local pods... 2026-07-29T08:55:04.894499281Z I0729 08:55:04.894460 3399 pods.go:178] Deleting pod: kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw 2026-07-29T08:55:04.896017047Z I0729 08:55:04.895986 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/9438e18bbad664a016701a3153a6b421e2d977be7eb0117f80bcf45ef1kgxvw, ips: 10.133.0.35 2026-07-29T08:55:04.896042596Z I0729 08:55:04.896032 3399 base_network_controller_policy.go:651] Processing NetworkPolicy kuadrant-system/kuadrant-operator-catalog-unpack-bundles to delete 1 local pods... 2026-07-29T08:55:11.765045821Z W0729 08:55:11.764993 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console 2026-07-29T08:55:15.581322557Z I0729 08:55:15.581272 3399 base_network_controller_pods.go:490] [default/kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx] creating logical port kuadrant-system_authorino-operator-75cc8d5cb7-pcvzx for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:15.581478167Z I0729 08:55:15.581439 3399 kube.go:234] Patching annotations on pod kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx 2026-07-29T08:55:15.600345516Z I0729 08:55:15.600293 3399 base_network_controller_pods.go:976] [default/kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx] addLogicalPort annotation time took 18.892086ms 2026-07-29T08:55:15.601503242Z I0729 08:55:15.601480 3399 pods.go:273] [kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx] addLogicalPort took 20.230625ms, libovsdb time 777.111µs 2026-07-29T08:55:15.944102277Z I0729 08:55:15.944056 3399 cniserver.go:311] [kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx d0604ed25ddeee12dcc68d27d52071a54a59f7e0f195c377a8d972d41dce377c network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:15.955336434Z I0729 08:55:15.955285 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: authorino-operator-75cc8d5cb7-pcvzx, hostIfaceName: d0604ed25ddeee1, network: default, NAD default, SandboxID: "d0604ed25ddeee12dcc68d27d52071a54a59f7e0f195c377a8d972d41dce377c", PCI device ID: , UID: "3943b20d-0422-4283-9b43-733489c7b6d1", MAC: 0a:58:0a:85:00:25, IPs: [10.133.0.37/23] 2026-07-29T08:55:16.006355510Z I0729 08:55:16.006311 3399 cniserver.go:300] [kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx d0604ed25ddeee12dcc68d27d52071a54a59f7e0f195c377a8d972d41dce377c network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/authorino-operator-75cc8d5cb7-pcvzx d0604ed25ddeee12dcc68d27d52071a54a59f7e0f195c377a8d972d41dce377c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:34:f9:e3:64:4f\",\"name\":\"d0604ed25ddeee1\"},{\"mac\":\"0a:58:0a:85:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ea5fd42-be8e-431c-8070-20be79263a18\"}],\"ips\":[{\"address\":\"10.133.0.37/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:17.999338215Z I0729 08:55:17.999290 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 2026-07-29T08:55:18.379938202Z I0729 08:55:18.379872 3399 base_network_controller_pods.go:490] [default/kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m] creating logical port kuadrant-system_limitador-operator-controller-manager-859d7b78bd-p4g2m for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:18.380160784Z I0729 08:55:18.380135 3399 kube.go:234] Patching annotations on pod kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 2026-07-29T08:55:18.395303434Z I0729 08:55:18.395265 3399 base_network_controller_pods.go:976] [default/kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m] addLogicalPort annotation time took 15.1694ms 2026-07-29T08:55:18.396592378Z I0729 08:55:18.396563 3399 pods.go:273] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m] addLogicalPort took 16.716023ms, libovsdb time 913.192µs 2026-07-29T08:55:18.749876017Z I0729 08:55:18.749831 3399 cniserver.go:311] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:18.761294880Z I0729 08:55:18.761251 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: limitador-operator-controller-manager-859d7b78bd-p4g2m, hostIfaceName: 42ddf7fea165abb, network: default, NAD default, SandboxID: "42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a", PCI device ID: , UID: "c3bbb00d-38d5-4517-8317-a6e13ebaa4fa", MAC: 0a:58:0a:85:00:26, IPs: [10.133.0.38/23] 2026-07-29T08:55:18.802793904Z W0729 08:55:18.802752 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/authorino-operator-metrics 2026-07-29T08:55:18.817378745Z I0729 08:55:18.817335 3399 cniserver.go:300] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:a9:a0:fc:cd:af\",\"name\":\"42ddf7fea165abb\"},{\"mac\":\"0a:58:0a:85:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e4942757-1cbe-4b77-8a43-83f1436b2e60\"}],\"ips\":[{\"address\":\"10.133.0.38/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:20.823955473Z W0729 08:55:20.823912 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/limitador-operator-metrics 2026-07-29T08:55:20.890497066Z I0729 08:55:20.890438 3399 base_network_controller_pods.go:490] [default/kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf] creating logical port kuadrant-system_dns-operator-controller-manager-864dd78c4c-qnwtf for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:20.890613404Z I0729 08:55:20.890593 3399 kube.go:234] Patching annotations on pod kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf 2026-07-29T08:55:20.918788787Z I0729 08:55:20.918754 3399 base_network_controller_pods.go:976] [default/kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf] addLogicalPort annotation time took 28.195373ms 2026-07-29T08:55:20.920052032Z I0729 08:55:20.920020 3399 pods.go:273] [kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf] addLogicalPort took 29.598889ms, libovsdb time 889.028µs 2026-07-29T08:55:21.252086915Z I0729 08:55:21.252040 3399 cniserver.go:311] [kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf 4a0e18586cb904a8685317dd801d427159ac3841d8147d66500083d8e3820733 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:21.261355395Z I0729 08:55:21.261309 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: dns-operator-controller-manager-864dd78c4c-qnwtf, hostIfaceName: 4a0e18586cb904a, network: default, NAD default, SandboxID: "4a0e18586cb904a8685317dd801d427159ac3841d8147d66500083d8e3820733", PCI device ID: , UID: "c6261e08-8c07-4a22-862a-5ed2c6534c38", MAC: 0a:58:0a:85:00:27, IPs: [10.133.0.39/23] 2026-07-29T08:55:21.316027160Z I0729 08:55:21.315977 3399 cniserver.go:300] [kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf 4a0e18586cb904a8685317dd801d427159ac3841d8147d66500083d8e3820733 network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/dns-operator-controller-manager-864dd78c4c-qnwtf 4a0e18586cb904a8685317dd801d427159ac3841d8147d66500083d8e3820733 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:ee:11:6f:52:2e\",\"name\":\"4a0e18586cb904a\"},{\"mac\":\"0a:58:0a:85:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b4b755ed-d9c8-4871-ad06-d501248cc630\"}],\"ips\":[{\"address\":\"10.133.0.39/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:21.895784784Z W0729 08:55:21.895738 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/limitador-operator-metrics 2026-07-29T08:55:21.909629587Z W0729 08:55:21.909551 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/limitador-operator-metrics 2026-07-29T08:55:23.841025949Z W0729 08:55:23.840970 3399 util.go:881] Failed to build global endpoints for port TCP/pprof/8082: empty IP address endpoints for service kuadrant-system/dns-operator-controller-manager-metrics-service 2026-07-29T08:55:23.841025949Z W0729 08:55:23.840992 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/dns-operator-controller-manager-metrics-service 2026-07-29T08:55:25.411516222Z I0729 08:55:25.411469 3399 base_network_controller_pods.go:490] [default/kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v] creating logical port kuadrant-system_kuadrant-operator-controller-manager-8684c9ff49-mqg5v for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:25.411826482Z I0729 08:55:25.411795 3399 kube.go:234] Patching annotations on pod kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 2026-07-29T08:55:25.437629954Z I0729 08:55:25.437593 3399 base_network_controller_pods.go:976] [default/kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v] addLogicalPort annotation time took 25.855638ms 2026-07-29T08:55:25.439038165Z I0729 08:55:25.439008 3399 pods.go:273] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v] addLogicalPort took 27.560936ms, libovsdb time 1.018068ms 2026-07-29T08:55:25.775941710Z I0729 08:55:25.775878 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:25.785443470Z I0729 08:55:25.785404 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: kuadrant-operator-controller-manager-8684c9ff49-mqg5v, hostIfaceName: 8e1904a18c21c6c, network: default, NAD default, SandboxID: "8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7", PCI device ID: , UID: "bb6d4c11-119a-4f5c-aefc-718baba74d17", MAC: 0a:58:0a:85:00:28, IPs: [10.133.0.40/23] 2026-07-29T08:55:25.841234534Z I0729 08:55:25.841184 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7 network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:c1:05:5d:84:81\",\"name\":\"8e1904a18c21c6c\"},{\"mac\":\"0a:58:0a:85:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/57d6dbf6-6b8a-41a2-b90a-f79c88142ee7\"}],\"ips\":[{\"address\":\"10.133.0.40/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:27.059372704Z W0729 08:55:27.059326 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/authorino-operator-metrics 2026-07-29T08:55:27.075373217Z W0729 08:55:27.075333 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/authorino-operator-metrics 2026-07-29T08:55:27.095101322Z W0729 08:55:27.095060 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/authorino-operator-metrics 2026-07-29T08:55:27.112130803Z W0729 08:55:27.112097 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/authorino-operator-metrics 2026-07-29T08:55:28.041141627Z I0729 08:55:28.041089 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=9 2026-07-29T08:55:29.816107647Z W0729 08:55:29.816064 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/authorino-operator-metrics 2026-07-29T08:55:30.877296181Z W0729 08:55:30.877256 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:31.825194006Z W0729 08:55:31.825140 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/limitador-operator-metrics 2026-07-29T08:55:33.371323303Z I0729 08:55:33.371284 3399 base_network_controller_pods.go:490] [default/kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7] creating logical port kuadrant-system_kuadrant-console-plugin-66b5774985-jstc7 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:33.371555169Z I0729 08:55:33.371529 3399 kube.go:234] Patching annotations on pod kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7 2026-07-29T08:55:33.384862264Z I0729 08:55:33.384816 3399 base_network_controller_pods.go:976] [default/kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7] addLogicalPort annotation time took 13.371876ms 2026-07-29T08:55:33.386533582Z I0729 08:55:33.386497 3399 pods.go:273] [kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7] addLogicalPort took 15.232973ms, libovsdb time 1.001046ms 2026-07-29T08:55:33.735138327Z I0729 08:55:33.735093 3399 cniserver.go:311] [kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7 650b4f33ed5a0819ef27337efb6895d02044adc8c34dcc572462a7ad124361cd network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:33.745866909Z I0729 08:55:33.745816 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: kuadrant-console-plugin-66b5774985-jstc7, hostIfaceName: 650b4f33ed5a081, network: default, NAD default, SandboxID: "650b4f33ed5a0819ef27337efb6895d02044adc8c34dcc572462a7ad124361cd", PCI device ID: , UID: "4325e682-4382-45e6-9fb0-0041ecb883e6", MAC: 0a:58:0a:85:00:29, IPs: [10.133.0.41/23] 2026-07-29T08:55:33.801062340Z I0729 08:55:33.801017 3399 cniserver.go:300] [kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7 650b4f33ed5a0819ef27337efb6895d02044adc8c34dcc572462a7ad124361cd network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/kuadrant-console-plugin-66b5774985-jstc7 650b4f33ed5a0819ef27337efb6895d02044adc8c34dcc572462a7ad124361cd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:c7:d0:87:91:b9\",\"name\":\"650b4f33ed5a081\"},{\"mac\":\"0a:58:0a:85:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f387821e-81a8-41e9-b0b9-78eff24c652c\"}],\"ips\":[{\"address\":\"10.133.0.41/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:34.841468471Z W0729 08:55:34.841427 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8082: empty IP address endpoints for service kuadrant-system/dns-operator-controller-manager-metrics-service 2026-07-29T08:55:34.841555352Z W0729 08:55:34.841524 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/dns-operator-controller-manager-metrics-service 2026-07-29T08:55:37.402009257Z I0729 08:55:37.401959 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-ff75677f7-msp7b 2026-07-29T08:55:37.402009257Z I0729 08:55:37.401996 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-ff75677f7-msp7b 2026-07-29T08:55:40.002756011Z W0729 08:55:40.002707 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:40.014466131Z W0729 08:55:40.014422 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:40.039197175Z W0729 08:55:40.039149 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:40.055500827Z W0729 08:55:40.055461 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:41.887032199Z W0729 08:55:41.886991 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:43.034844154Z I0729 08:55:43.034796 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 2026-07-29T08:55:43.722690083Z I0729 08:55:43.722654 3399 pods.go:178] Deleting pod: kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 2026-07-29T08:55:43.725806830Z I0729 08:55:43.725767 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v, ips: 10.133.0.40 2026-07-29T08:55:43.737748605Z I0729 08:55:43.737710 3399 base_network_controller_pods.go:490] [default/kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f] creating logical port kuadrant-system_kuadrant-operator-controller-manager-8684c9ff49-sr56f for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:43.737960643Z I0729 08:55:43.737940 3399 kube.go:234] Patching annotations on pod kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 2026-07-29T08:55:43.759350757Z I0729 08:55:43.759299 3399 base_network_controller_pods.go:976] [default/kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f] addLogicalPort annotation time took 21.42915ms 2026-07-29T08:55:43.760966209Z I0729 08:55:43.760929 3399 pods.go:273] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f] addLogicalPort took 23.233892ms, libovsdb time 1.066918ms 2026-07-29T08:55:43.764030448Z I0729 08:55:43.763999 3399 pods.go:178] Deleting pod: kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 2026-07-29T08:55:43.769386730Z I0729 08:55:43.769352 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m, ips: 10.133.0.38 2026-07-29T08:55:43.782232839Z I0729 08:55:43.781501 3399 base_network_controller_pods.go:490] [default/kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2] creating logical port kuadrant-system_limitador-operator-controller-manager-859d7b78bd-h57d2 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:55:43.782232839Z I0729 08:55:43.781674 3399 kube.go:234] Patching annotations on pod kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2 2026-07-29T08:55:43.794273272Z I0729 08:55:43.794230 3399 base_network_controller_pods.go:976] [default/kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2] addLogicalPort annotation time took 12.603308ms 2026-07-29T08:55:43.796853714Z I0729 08:55:43.796815 3399 pods.go:273] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2] addLogicalPort took 15.328539ms, libovsdb time 1.999686ms 2026-07-29T08:55:48.470732160Z W0729 08:55:48.470679 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:53.899267688Z I0729 08:55:53.899217 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=694 2026-07-29T08:55:54.891437261Z I0729 08:55:54.891386 3399 cniserver.go:311] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:54.897726747Z I0729 08:55:54.897669 3399 cniserver.go:311] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2 78fdd1fdfd317dba4fd61bcb1049a428fe1a9d641d4becfaeab5ebbc0debf016 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:54.898503644Z I0729 08:55:54.898468 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:55:54.902795670Z I0729 08:55:54.902754 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:54.917087180Z I0729 08:55:54.917022 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: limitador-operator-controller-manager-859d7b78bd-h57d2, hostIfaceName: 78fdd1fdfd317db, network: default, NAD default, SandboxID: "78fdd1fdfd317dba4fd61bcb1049a428fe1a9d641d4becfaeab5ebbc0debf016", PCI device ID: , UID: "0bb85cb3-2f26-4396-9ac7-68cdb7a08f89", MAC: 0a:58:0a:85:00:2b, IPs: [10.133.0.43/23] 2026-07-29T08:55:54.936596529Z I0729 08:55:54.936540 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: kuadrant-operator-controller-manager-8684c9ff49-sr56f, hostIfaceName: 6f34e183facbda3, network: default, NAD default, SandboxID: "6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd", PCI device ID: , UID: "2569b9e7-cdd8-4ae2-ade8-945c41939495", MAC: 0a:58:0a:85:00:2a, IPs: [10.133.0.42/23] 2026-07-29T08:55:54.942129236Z E0729 08:55:54.942088 3399 link_network_manager.go:104] Error while syncing link "42ddf7fea165abb": failed to get address from link "42ddf7fea165abb": failed to list addresses for "42ddf7fea165abb": no such device 2026-07-29T08:55:54.943931824Z E0729 08:55:54.943887 3399 link_network_manager.go:104] Error while syncing link "42ddf7fea165abb": failed to get address from link "42ddf7fea165abb": failed to list addresses for "42ddf7fea165abb": no such device 2026-07-29T08:55:54.977964020Z E0729 08:55:54.977925 3399 link_network_manager.go:104] Error while syncing link "8e1904a18c21c6c": failed to get address from link "8e1904a18c21c6c": failed to list addresses for "8e1904a18c21c6c": no such device 2026-07-29T08:55:55.018783457Z I0729 08:55:55.018728 3399 cniserver.go:300] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-p4g2m 42ddf7fea165abbea2bac15acaed35111236ff22e884bbf595140ca6caa1d90a network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:55.025567711Z I0729 08:55:55.025512 3399 cniserver.go:300] [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2 78fdd1fdfd317dba4fd61bcb1049a428fe1a9d641d4becfaeab5ebbc0debf016 network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/limitador-operator-controller-manager-859d7b78bd-h57d2 78fdd1fdfd317dba4fd61bcb1049a428fe1a9d641d4becfaeab5ebbc0debf016 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:17:1f:29:83:7c\",\"name\":\"78fdd1fdfd317db\"},{\"mac\":\"0a:58:0a:85:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d49c9404-fad2-4152-8e96-f3ca7f997637\"}],\"ips\":[{\"address\":\"10.133.0.43/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:55.035106264Z I0729 08:55:55.034980 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7 network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-mqg5v 8e1904a18c21c6c35cff21be28de4d5877799cedd1020b7dc2a196dc1dace5f7 network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:55.057623164Z I0729 08:55:55.057571 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:dc:2c:28:95:21\",\"name\":\"6f34e183facbda3\"},{\"mac\":\"0a:58:0a:85:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3c25eb7d-13f3-4e54-a1b4-46e80eabd738\"}],\"ips\":[{\"address\":\"10.133.0.42/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:55:56.061807665Z W0729 08:55:56.061766 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:56.099137544Z W0729 08:55:56.099088 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/limitador-operator-metrics 2026-07-29T08:55:58.787521753Z W0729 08:55:58.787469 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:58.794106311Z W0729 08:55:58.794062 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:58.806513939Z W0729 08:55:58.806470 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:58.813958571Z W0729 08:55:58.813923 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:59.504474350Z W0729 08:55:59.504417 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:59.504474350Z W0729 08:55:59.504442 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T08:55:59.684093122Z I0729 08:55:59.684047 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:55:59.707480583Z E0729 08:55:59.707446 3399 link_network_manager.go:104] Error while syncing link "6f34e183facbda3": failed to get address from link "6f34e183facbda3": failed to list addresses for "6f34e183facbda3": no such device 2026-07-29T08:55:59.756909252Z I0729 08:55:59.756848 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 6f34e183facbda3b12536f157a3a544c788e7b39bb6d60ddaaecdfbca163a1cd network default NAD default NAD key default], result "{}", err 2026-07-29T08:55:59.812162417Z I0729 08:55:59.812120 3399 pods.go:178] Deleting pod: kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 2026-07-29T08:55:59.813566308Z I0729 08:55:59.813536 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f, ips: 10.133.0.42 2026-07-29T08:55:59.898974546Z I0729 08:55:59.898923 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=138 2026-07-29T08:56:00.093920589Z I0729 08:56:00.093858 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 2026-07-29T08:56:00.093960795Z I0729 08:56:00.093925 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/kuadrant-operator-controller-manager-8684c9ff49-sr56f 2026-07-29T08:56:01.040321278Z I0729 08:56:01.040278 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=9 2026-07-29T08:56:07.075260924Z W0729 08:56:07.075214 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/limitador-operator-metrics 2026-07-29T08:56:15.947606554Z I0729 08:56:15.947527 3399 base_network_controller_pods.go:490] [default/openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs] creating logical port openshift-ingress_maas-default-gateway-openshift-default-5ccb968c66-lqjvs for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:56:15.947769028Z I0729 08:56:15.947747 3399 kube.go:234] Patching annotations on pod openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs 2026-07-29T08:56:15.965758237Z I0729 08:56:15.965700 3399 base_network_controller_pods.go:976] [default/openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs] addLogicalPort annotation time took 17.993673ms 2026-07-29T08:56:15.967228930Z I0729 08:56:15.967192 3399 pods.go:273] [openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs] addLogicalPort took 19.687104ms, libovsdb time 875.578µs 2026-07-29T08:56:16.517794138Z I0729 08:56:16.517746 3399 cniserver.go:311] [openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs 3324360f6599167d7ba2291597cb8ec697e4afb672fc60fc924a24d3b191460e network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:56:16.527334903Z I0729 08:56:16.527283 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: maas-default-gateway-openshift-default-5ccb968c66-lqjvs, hostIfaceName: 3324360f6599167, network: default, NAD default, SandboxID: "3324360f6599167d7ba2291597cb8ec697e4afb672fc60fc924a24d3b191460e", PCI device ID: , UID: "45bd778b-da1b-451a-8d60-f66d883aadcf", MAC: 0a:58:0a:85:00:2c, IPs: [10.133.0.44/23] 2026-07-29T08:56:16.591390778Z I0729 08:56:16.591345 3399 cniserver.go:300] [openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs 3324360f6599167d7ba2291597cb8ec697e4afb672fc60fc924a24d3b191460e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/maas-default-gateway-openshift-default-5ccb968c66-lqjvs 3324360f6599167d7ba2291597cb8ec697e4afb672fc60fc924a24d3b191460e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:9b:1d:80:4c:46\",\"name\":\"3324360f6599167\"},{\"mac\":\"0a:58:0a:85:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cdbd9b14-a22f-4461-a514-fa28fa4a8bb1\"}],\"ips\":[{\"address\":\"10.133.0.44/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:56:16.605454326Z I0729 08:56:16.605406 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/kuadrant-operator-controller-manager-6fb7879bc9-pnfsv 2026-07-29T08:56:16.605454326Z I0729 08:56:16.605431 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/kuadrant-operator-controller-manager-6fb7879bc9-pnfsv 2026-07-29T08:56:20.029693516Z I0729 08:56:20.029647 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 2026-07-29T08:56:20.209460236Z W0729 08:56:20.209413 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:56:20.209497290Z W0729 08:56:20.209457 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:56:21.215175234Z W0729 08:56:21.215109 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:56:21.215175234Z W0729 08:56:21.215136 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:56:30.899145154Z I0729 08:56:30.899099 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=15 2026-07-29T08:56:35.347129810Z I0729 08:56:35.347088 3399 base_network_controller_pods.go:490] [default/kuadrant-system/limitador-limitador-64c94fd64f-rj9tr] creating logical port kuadrant-system_limitador-limitador-64c94fd64f-rj9tr for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:56:35.347275159Z I0729 08:56:35.347258 3399 kube.go:234] Patching annotations on pod kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 2026-07-29T08:56:35.357508936Z I0729 08:56:35.357465 3399 base_network_controller_pods.go:976] [default/kuadrant-system/limitador-limitador-64c94fd64f-rj9tr] addLogicalPort annotation time took 10.251596ms 2026-07-29T08:56:35.359264506Z I0729 08:56:35.359224 3399 pods.go:273] [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr] addLogicalPort took 12.154043ms, libovsdb time 1.037403ms 2026-07-29T08:56:35.814698807Z I0729 08:56:35.814654 3399 cniserver.go:311] [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:56:35.826399697Z I0729 08:56:35.826352 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: limitador-limitador-64c94fd64f-rj9tr, hostIfaceName: 78b77a8b5a7bf18, network: default, NAD default, SandboxID: "78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a", PCI device ID: , UID: "40e4d223-c51d-4536-8bde-9cb4635dc6c6", MAC: 0a:58:0a:85:00:2d, IPs: [10.133.0.45/23] 2026-07-29T08:56:35.882405601Z I0729 08:56:35.882360 3399 cniserver.go:300] [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:00:5c:65:67:bc\",\"name\":\"78b77a8b5a7bf18\"},{\"mac\":\"0a:58:0a:85:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd7520be-c7c4-47c9-bdd4-b2ed14ff26be\"}],\"ips\":[{\"address\":\"10.133.0.45/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:56:36.157092025Z I0729 08:56:36.157049 3399 base_network_controller_pods.go:490] [default/kuadrant-system/authorino-58cd955675-m4wfv] creating logical port kuadrant-system_authorino-58cd955675-m4wfv for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:56:36.157272030Z I0729 08:56:36.157253 3399 kube.go:234] Patching annotations on pod kuadrant-system/authorino-58cd955675-m4wfv 2026-07-29T08:56:36.168007036Z I0729 08:56:36.167962 3399 base_network_controller_pods.go:976] [default/kuadrant-system/authorino-58cd955675-m4wfv] addLogicalPort annotation time took 10.736231ms 2026-07-29T08:56:36.170402399Z I0729 08:56:36.170364 3399 pods.go:273] [kuadrant-system/authorino-58cd955675-m4wfv] addLogicalPort took 13.334551ms, libovsdb time 1.856028ms 2026-07-29T08:56:36.521302235Z I0729 08:56:36.521258 3399 cniserver.go:311] [kuadrant-system/authorino-58cd955675-m4wfv a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:56:36.532670653Z I0729 08:56:36.532627 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: authorino-58cd955675-m4wfv, hostIfaceName: a8b9ea3e8d37654, network: default, NAD default, SandboxID: "a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f", PCI device ID: , UID: "d91f6036-65f4-43e2-beaa-c6ab3b83ad20", MAC: 0a:58:0a:85:00:2e, IPs: [10.133.0.46/23] 2026-07-29T08:56:36.588068854Z I0729 08:56:36.588017 3399 cniserver.go:300] [kuadrant-system/authorino-58cd955675-m4wfv a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/authorino-58cd955675-m4wfv a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:e1:10:55:d0:ac\",\"name\":\"a8b9ea3e8d37654\"},{\"mac\":\"0a:58:0a:85:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/076a7653-71ae-4ba1-aa3b-fa9dbc238225\"}],\"ips\":[{\"address\":\"10.133.0.46/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:56:43.482481115Z I0729 08:56:43.482432 3399 cniserver.go:311] [kuadrant-system/authorino-58cd955675-m4wfv a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:56:43.516395353Z E0729 08:56:43.516360 3399 link_network_manager.go:104] Error while syncing link "a8b9ea3e8d37654": failed to get address from link "a8b9ea3e8d37654": failed to list addresses for "a8b9ea3e8d37654": no such device 2026-07-29T08:56:43.558562914Z I0729 08:56:43.558518 3399 cniserver.go:300] [kuadrant-system/authorino-58cd955675-m4wfv a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/authorino-58cd955675-m4wfv a8b9ea3e8d376548caab213a63b714bf9e64bfbf960a1dbbaca06fc850ba5a4f network default NAD default NAD key default], result "{}", err 2026-07-29T08:56:43.596979993Z I0729 08:56:43.596935 3399 pods.go:178] Deleting pod: kuadrant-system/authorino-58cd955675-m4wfv 2026-07-29T08:56:43.598661498Z I0729 08:56:43.598622 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/authorino-58cd955675-m4wfv, ips: 10.133.0.46 2026-07-29T08:56:44.357075373Z I0729 08:56:44.357027 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/authorino-58cd955675-m4wfv 2026-07-29T08:56:44.357075373Z I0729 08:56:44.357060 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/authorino-58cd955675-m4wfv 2026-07-29T08:56:53.240015620Z I0729 08:56:53.239971 3399 cniserver.go:311] [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:56:53.274316956Z E0729 08:56:53.274278 3399 link_network_manager.go:104] Error while syncing link "78b77a8b5a7bf18": failed to get address from link "78b77a8b5a7bf18": failed to list addresses for "78b77a8b5a7bf18": no such device 2026-07-29T08:56:53.322066694Z I0729 08:56:53.322022 3399 cniserver.go:300] [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 78b77a8b5a7bf18ab653ff8c9bf85eaab39b4d449f1ebe44d6c845dd7231217a network default NAD default NAD key default], result "{}", err 2026-07-29T08:56:53.362559149Z I0729 08:56:53.362521 3399 pods.go:178] Deleting pod: kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 2026-07-29T08:56:53.364277304Z I0729 08:56:53.364235 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/limitador-limitador-64c94fd64f-rj9tr, ips: 10.133.0.45 2026-07-29T08:56:53.711248155Z I0729 08:56:53.711197 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 2026-07-29T08:56:53.711286453Z I0729 08:56:53.711264 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/limitador-limitador-64c94fd64f-rj9tr 2026-07-29T08:56:57.688530298Z I0729 08:56:57.688481 3399 namespace.go:89] [odh-ai-gateway-infra] adding namespace 2026-07-29T08:56:57.689191232Z I0729 08:56:57.689158 3399 namespace.go:93] [odh-ai-gateway-infra] adding namespace took 646.298µs 2026-07-29T08:56:57.706767462Z I0729 08:56:57.706732 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T08:56:57.714861076Z I0729 08:56:57.714826 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T08:56:58.179634849Z I0729 08:56:58.179595 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn] creating logical port odh-ai-gateway-infra_postgres-69fb4594cf-p9mfn for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:56:58.179828673Z I0729 08:56:58.179792 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn 2026-07-29T08:56:58.190361568Z I0729 08:56:58.190318 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn] addLogicalPort annotation time took 10.553739ms 2026-07-29T08:56:58.192064986Z I0729 08:56:58.192021 3399 pods.go:273] [odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn] addLogicalPort took 12.440888ms, libovsdb time 1.0729ms 2026-07-29T08:56:58.545303112Z I0729 08:56:58.545261 3399 cniserver.go:311] [odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn d6dd0e74217ee162a473a6b079fd0355df27bd5c39d099dd7a773fb2fdf2a669 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:56:58.554928364Z I0729 08:56:58.554860 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: postgres-69fb4594cf-p9mfn, hostIfaceName: d6dd0e74217ee16, network: default, NAD default, SandboxID: "d6dd0e74217ee162a473a6b079fd0355df27bd5c39d099dd7a773fb2fdf2a669", PCI device ID: , UID: "bdf5f625-1ed1-4e03-9dd0-f9ddbe59c1d2", MAC: 0a:58:0a:85:00:2f, IPs: [10.133.0.47/23] 2026-07-29T08:56:58.610194086Z I0729 08:56:58.610152 3399 cniserver.go:300] [odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn d6dd0e74217ee162a473a6b079fd0355df27bd5c39d099dd7a773fb2fdf2a669 network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/postgres-69fb4594cf-p9mfn d6dd0e74217ee162a473a6b079fd0355df27bd5c39d099dd7a773fb2fdf2a669 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:45:c8:26:80:6a\",\"name\":\"d6dd0e74217ee16\"},{\"mac\":\"0a:58:0a:85:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd73508b-2344-4dd5-b320-9f7fbc8243b3\"}],\"ips\":[{\"address\":\"10.133.0.47/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:03.475118574Z W0729 08:57:03.475075 3399 util.go:881] Failed to build global endpoints for port TCP//5432: empty IP address endpoints for service odh-ai-gateway-infra/postgres 2026-07-29T08:57:09.508773614Z W0729 08:57:09.508714 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 5432: empty IP address endpoints for service odh-ai-gateway-infra/postgres 2026-07-29T08:57:10.008402461Z I0729 08:57:10.008348 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressIP" totalItems=9 2026-07-29T08:57:10.023386116Z I0729 08:57:10.023345 3399 base_network_controller_pods.go:490] [default/kuadrant-system/authorino-5d4b67c446-t79mc] creating logical port kuadrant-system_authorino-5d4b67c446-t79mc for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:10.023486584Z I0729 08:57:10.023469 3399 kube.go:234] Patching annotations on pod kuadrant-system/authorino-5d4b67c446-t79mc 2026-07-29T08:57:10.032567091Z I0729 08:57:10.032525 3399 base_network_controller_pods.go:976] [default/kuadrant-system/authorino-5d4b67c446-t79mc] addLogicalPort annotation time took 9.092051ms 2026-07-29T08:57:10.034190358Z I0729 08:57:10.034163 3399 pods.go:273] [kuadrant-system/authorino-5d4b67c446-t79mc] addLogicalPort took 10.836129ms, libovsdb time 895.463µs 2026-07-29T08:57:10.286932024Z I0729 08:57:10.286868 3399 base_network_controller_pods.go:490] [default/kuadrant-system/authorino-7686dc7b54-948j9] creating logical port kuadrant-system_authorino-7686dc7b54-948j9 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:10.287171536Z I0729 08:57:10.287151 3399 kube.go:234] Patching annotations on pod kuadrant-system/authorino-7686dc7b54-948j9 2026-07-29T08:57:10.297185239Z I0729 08:57:10.297140 3399 base_network_controller_pods.go:976] [default/kuadrant-system/authorino-7686dc7b54-948j9] addLogicalPort annotation time took 10.031984ms 2026-07-29T08:57:10.298595412Z I0729 08:57:10.298563 3399 pods.go:273] [kuadrant-system/authorino-7686dc7b54-948j9] addLogicalPort took 11.714708ms, libovsdb time 929.38µs 2026-07-29T08:57:10.408083011Z I0729 08:57:10.408040 3399 base_network_controller_pods.go:490] [default/kuadrant-system/authorino-645fdb8cd6-kjt89] creating logical port kuadrant-system_authorino-645fdb8cd6-kjt89 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:10.408261375Z I0729 08:57:10.408239 3399 kube.go:234] Patching annotations on pod kuadrant-system/authorino-645fdb8cd6-kjt89 2026-07-29T08:57:10.420074581Z I0729 08:57:10.420020 3399 base_network_controller_pods.go:976] [default/kuadrant-system/authorino-645fdb8cd6-kjt89] addLogicalPort annotation time took 11.823599ms 2026-07-29T08:57:10.421545709Z I0729 08:57:10.421512 3399 pods.go:273] [kuadrant-system/authorino-645fdb8cd6-kjt89] addLogicalPort took 13.491568ms, libovsdb time 1.050949ms 2026-07-29T08:57:10.773877577Z I0729 08:57:10.773834 3399 cniserver.go:311] [kuadrant-system/authorino-645fdb8cd6-kjt89 b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:57:10.783305658Z I0729 08:57:10.783259 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: authorino-645fdb8cd6-kjt89, hostIfaceName: b7ea56da285408d, network: default, NAD default, SandboxID: "b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa", PCI device ID: , UID: "4c77e77f-471c-4325-abe9-a6a82a713831", MAC: 0a:58:0a:85:00:32, IPs: [10.133.0.50/23] 2026-07-29T08:57:10.839409979Z I0729 08:57:10.839357 3399 cniserver.go:300] [kuadrant-system/authorino-645fdb8cd6-kjt89 b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/authorino-645fdb8cd6-kjt89 b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:fd:b9:fb:2c:91\",\"name\":\"b7ea56da285408d\"},{\"mac\":\"0a:58:0a:85:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b19fb684-15ee-452b-81eb-0754c42306e0\"}],\"ips\":[{\"address\":\"10.133.0.50/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:11.524158369Z I0729 08:57:11.524113 3399 pods.go:178] Deleting pod: kuadrant-system/authorino-5d4b67c446-t79mc 2026-07-29T08:57:11.525773417Z I0729 08:57:11.525725 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/authorino-5d4b67c446-t79mc, ips: 10.133.0.48 2026-07-29T08:57:11.538054181Z I0729 08:57:11.538019 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/authorino-5d4b67c446-t79mc 2026-07-29T08:57:11.538126574Z I0729 08:57:11.538098 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/authorino-5d4b67c446-t79mc 2026-07-29T08:57:11.550877088Z I0729 08:57:11.550799 3399 pods.go:178] Deleting pod: kuadrant-system/authorino-7686dc7b54-948j9 2026-07-29T08:57:11.552325520Z I0729 08:57:11.552293 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/authorino-7686dc7b54-948j9, ips: 10.133.0.49 2026-07-29T08:57:11.562287256Z I0729 08:57:11.562253 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/authorino-7686dc7b54-948j9 2026-07-29T08:57:11.562344825Z I0729 08:57:11.562331 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/authorino-7686dc7b54-948j9 2026-07-29T08:57:11.871916228Z I0729 08:57:11.871860 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/authorino-75d556bd77-xm84v 2026-07-29T08:57:11.871916228Z I0729 08:57:11.871910 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/authorino-75d556bd77-xm84v 2026-07-29T08:57:12.024321159Z I0729 08:57:12.024281 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=10 2026-07-29T08:57:12.898959060Z I0729 08:57:12.898916 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=39 2026-07-29T08:57:16.218045027Z I0729 08:57:16.217995 3399 base_network_controller_pods.go:490] [default/opendatahub/maas-controller-78dc94c995-bqjqw] creating logical port opendatahub_maas-controller-78dc94c995-bqjqw for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:16.218145159Z I0729 08:57:16.218127 3399 kube.go:234] Patching annotations on pod opendatahub/maas-controller-78dc94c995-bqjqw 2026-07-29T08:57:16.227973572Z I0729 08:57:16.227933 3399 base_network_controller_policy.go:1190] Adding network policy opendatahub/maas-controller-allow-monitoring for network default 2026-07-29T08:57:16.228626976Z I0729 08:57:16.228590 3399 base_network_controller_policy.go:1085] Policy opendatahub/maas-controller-allow-monitoring added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: redhat-ods-monitoring,},}_LS{}_LNM] 2026-07-29T08:57:16.231325304Z I0729 08:57:16.231295 3399 base_network_controller_pods.go:976] [default/opendatahub/maas-controller-78dc94c995-bqjqw] addLogicalPort annotation time took 13.215203ms 2026-07-29T08:57:16.232569509Z I0729 08:57:16.232544 3399 pods.go:273] [opendatahub/maas-controller-78dc94c995-bqjqw] addLogicalPort took 14.572239ms, libovsdb time 725.677µs 2026-07-29T08:57:16.233457734Z I0729 08:57:16.233432 3399 base_network_controller_policy.go:1243] Create network policy opendatahub/maas-controller-allow-monitoring resources completed, update namespace loglevel 2026-07-29T08:57:16.377112711Z I0729 08:57:16.377065 3399 base_network_controller_pods.go:490] [default/opendatahub/maas-controller-95cc89ddf-sdb9x] creating logical port opendatahub_maas-controller-95cc89ddf-sdb9x for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:16.377217133Z I0729 08:57:16.377202 3399 kube.go:234] Patching annotations on pod opendatahub/maas-controller-95cc89ddf-sdb9x 2026-07-29T08:57:16.389911974Z I0729 08:57:16.389848 3399 base_network_controller_pods.go:976] [default/opendatahub/maas-controller-95cc89ddf-sdb9x] addLogicalPort annotation time took 12.592316ms 2026-07-29T08:57:16.391371805Z I0729 08:57:16.391345 3399 pods.go:273] [opendatahub/maas-controller-95cc89ddf-sdb9x] addLogicalPort took 14.301995ms, libovsdb time 785.826µs 2026-07-29T08:57:16.823532384Z I0729 08:57:16.823485 3399 cniserver.go:311] [opendatahub/maas-controller-78dc94c995-bqjqw 91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:57:16.834174558Z I0729 08:57:16.834114 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: maas-controller-78dc94c995-bqjqw, hostIfaceName: 91540e3e143103f, network: default, NAD default, SandboxID: "91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e", PCI device ID: , UID: "89f7d855-4de0-4c25-9b87-372bb8a1adff", MAC: 0a:58:0a:85:00:33, IPs: [10.133.0.51/23] 2026-07-29T08:57:16.854676127Z I0729 08:57:16.854630 3399 cniserver.go:311] [opendatahub/maas-controller-95cc89ddf-sdb9x 8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:57:16.868052463Z I0729 08:57:16.868001 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: maas-controller-95cc89ddf-sdb9x, hostIfaceName: 8055bc93172dc47, network: default, NAD default, SandboxID: "8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f", PCI device ID: , UID: "e55581bb-2fde-4fa2-bbd1-3b53ed820e65", MAC: 0a:58:0a:85:00:34, IPs: [10.133.0.52/23] 2026-07-29T08:57:16.899232366Z I0729 08:57:16.899166 3399 cniserver.go:300] [opendatahub/maas-controller-78dc94c995-bqjqw 91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e network default NAD default NAD key default] ADD finished CNI request [opendatahub/maas-controller-78dc94c995-bqjqw 91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:40:f3:dd:d8:fb\",\"name\":\"91540e3e143103f\"},{\"mac\":\"0a:58:0a:85:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/278d4ec7-b7c4-4e3b-9db7-81b2a771271a\"}],\"ips\":[{\"address\":\"10.133.0.51/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:16.933098400Z I0729 08:57:16.933041 3399 cniserver.go:300] [opendatahub/maas-controller-95cc89ddf-sdb9x 8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f network default NAD default NAD key default] ADD finished CNI request [opendatahub/maas-controller-95cc89ddf-sdb9x 8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:46:98:c5:9f:e7\",\"name\":\"8055bc93172dc47\"},{\"mac\":\"0a:58:0a:85:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4dac9834-f6b6-49a2-a6c0-5cf59d3db698\"}],\"ips\":[{\"address\":\"10.133.0.52/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:19.722463301Z I0729 08:57:19.722417 3399 namespace.go:89] [ai-tenants] adding namespace 2026-07-29T08:57:19.723267162Z I0729 08:57:19.723228 3399 namespace.go:93] [ai-tenants] adding namespace took 719.839µs 2026-07-29T08:57:19.732775467Z I0729 08:57:19.732745 3399 namespace.go:133] [ai-tenants] updating namespace 2026-07-29T08:57:19.742029269Z I0729 08:57:19.742001 3399 namespace.go:133] [ai-tenants] updating namespace 2026-07-29T08:57:19.773370027Z I0729 08:57:19.773315 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T08:57:20.579024828Z W0729 08:57:20.578979 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T08:57:20.579990599Z W0729 08:57:20.579958 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T08:57:20.593932066Z W0729 08:57:20.593810 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T08:57:20.596970981Z W0729 08:57:20.596934 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T08:57:28.019631093Z I0729 08:57:28.019578 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=11 2026-07-29T08:57:30.898258488Z I0729 08:57:30.898223 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=86 2026-07-29T08:57:31.589382481Z W0729 08:57:31.589331 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T08:57:31.589515185Z W0729 08:57:31.589479 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T08:57:31.780786655Z I0729 08:57:31.780744 3399 cniserver.go:311] [opendatahub/maas-controller-78dc94c995-bqjqw 91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e network default NAD default NAD key default] DEL starting CNI request 2026-07-29T08:57:31.802388255Z E0729 08:57:31.802350 3399 link_network_manager.go:104] Error while syncing link "91540e3e143103f": failed to get address from link "91540e3e143103f": failed to list addresses for "91540e3e143103f": no such device 2026-07-29T08:57:31.846272436Z I0729 08:57:31.846224 3399 cniserver.go:300] [opendatahub/maas-controller-78dc94c995-bqjqw 91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e network default NAD default NAD key default] DEL finished CNI request [opendatahub/maas-controller-78dc94c995-bqjqw 91540e3e143103fcf02f0464ca11a4d957c276402f5d69bceb5fba0e44ebcd3e network default NAD default NAD key default], result "{}", err 2026-07-29T08:57:31.887982580Z I0729 08:57:31.887946 3399 pods.go:178] Deleting pod: opendatahub/maas-controller-78dc94c995-bqjqw 2026-07-29T08:57:31.889843913Z I0729 08:57:31.889809 3399 pods.go:238] Attempting to release IPs for pod: opendatahub/maas-controller-78dc94c995-bqjqw, ips: 10.133.0.51 2026-07-29T08:57:31.889906817Z I0729 08:57:31.889864 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T08:57:31.890205284Z I0729 08:57:31.890174 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/maas-controller-allow-monitoring to delete 1 local pods... 2026-07-29T08:57:32.644509277Z I0729 08:57:32.644462 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/maas-controller-78dc94c995-bqjqw 2026-07-29T08:57:32.644544450Z I0729 08:57:32.644511 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/maas-controller-78dc94c995-bqjqw 2026-07-29T08:57:32.644544450Z I0729 08:57:32.644523 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/maas-controller-78dc94c995-bqjqw 2026-07-29T08:57:32.644569465Z I0729 08:57:32.644556 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/maas-controller-78dc94c995-bqjqw 2026-07-29T08:57:53.526300931Z I0729 08:57:53.526252 3399 namespace.go:89] [models-as-a-service] adding namespace 2026-07-29T08:57:53.527228261Z I0729 08:57:53.527189 3399 namespace.go:93] [models-as-a-service] adding namespace took 869.986µs 2026-07-29T08:57:53.543189507Z I0729 08:57:53.543152 3399 namespace.go:133] [models-as-a-service] updating namespace 2026-07-29T08:57:53.557424292Z I0729 08:57:53.557388 3399 namespace.go:133] [models-as-a-service] updating namespace 2026-07-29T08:57:54.315516918Z I0729 08:57:54.315478 3399 base_network_controller_policy.go:1190] Adding network policy odh-ai-gateway-infra/maas-api-cleanup-restrict for network default 2026-07-29T08:57:54.316358408Z I0729 08:57:54.316318 3399 base_network_controller_policy.go:1085] Policy odh-ai-gateway-infra/maas-api-cleanup-restrict added to peer address sets [default-network-controller_odh-ai-gateway-infra_LS{ML:{app.kubernetes.io/component: api,app.kubernetes.io/name: maas-api,app.kubernetes.io/part-of: models-as-a-service,},}] 2026-07-29T08:57:54.319355719Z I0729 08:57:54.319315 3399 base_network_controller_policy.go:1243] Create network policy odh-ai-gateway-infra/maas-api-cleanup-restrict resources completed, update namespace loglevel 2026-07-29T08:57:54.333690294Z I0729 08:57:54.333621 3399 base_network_controller_policy.go:1190] Adding network policy odh-ai-gateway-infra/maas-api-allow-cleanup for network default 2026-07-29T08:57:54.334247999Z I0729 08:57:54.334222 3399 base_network_controller_policy.go:1085] Policy odh-ai-gateway-infra/maas-api-allow-cleanup added to peer address sets [default-network-controller_odh-ai-gateway-infra_LS{ML:{app: maas-api-cleanup,app.kubernetes.io/component: api,app.kubernetes.io/name: maas-api,app.kubernetes.io/part-of: models-as-a-service,},}] 2026-07-29T08:57:54.334875186Z I0729 08:57:54.334853 3399 base_network_controller_policy.go:1243] Create network policy odh-ai-gateway-infra/maas-api-allow-cleanup resources completed, update namespace loglevel 2026-07-29T08:57:54.561928828Z I0729 08:57:54.561876 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j] creating logical port odh-ai-gateway-infra_maas-api-84c7dbbd44-spd7j for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:54.562295239Z I0729 08:57:54.562102 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j 2026-07-29T08:57:54.574308105Z I0729 08:57:54.574270 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j] addLogicalPort annotation time took 12.228243ms 2026-07-29T08:57:54.576110934Z I0729 08:57:54.576078 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j] addLogicalPort took 14.221622ms, libovsdb time 1.178788ms 2026-07-29T08:57:54.712211027Z I0729 08:57:54.712143 3399 base_network_controller_policy.go:1190] Adding network policy odh-ai-gateway-infra/maas-authorino-allow for network default 2026-07-29T08:57:54.712854275Z I0729 08:57:54.712826 3399 base_network_controller_policy.go:1085] Policy odh-ai-gateway-infra/maas-authorino-allow added to peer address sets [default-network-controller_LS{ME:{LSR{Key:kubernetes.io/metadata.name,Operator:In,Values:[kuadrant-system openshift-operators rh-connectivity-link],}},}_LS{ML:{authorino-resource: authorino,},}] 2026-07-29T08:57:54.714332330Z I0729 08:57:54.714299 3399 base_network_controller_policy.go:1243] Create network policy odh-ai-gateway-infra/maas-authorino-allow resources completed, update namespace loglevel 2026-07-29T08:57:54.737602031Z I0729 08:57:54.737553 3399 base_network_controller_policy.go:1190] Adding network policy odh-ai-gateway-infra/maas-api-allow-deployment-ns for network default 2026-07-29T08:57:54.738126581Z I0729 08:57:54.738089 3399 base_network_controller_policy.go:1085] Policy odh-ai-gateway-infra/maas-api-allow-deployment-ns added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: opendatahub,},}_LS{}_LNM] 2026-07-29T08:57:54.739185102Z I0729 08:57:54.739155 3399 base_network_controller_policy.go:1243] Create network policy odh-ai-gateway-infra/maas-api-allow-deployment-ns resources completed, update namespace loglevel 2026-07-29T08:57:54.764218193Z I0729 08:57:54.764170 3399 base_network_controller_policy.go:1190] Adding network policy odh-ai-gateway-infra/maas-api-allow-gateway for network default 2026-07-29T08:57:54.764775646Z I0729 08:57:54.764741 3399 base_network_controller_policy.go:1085] Policy odh-ai-gateway-infra/maas-api-allow-gateway added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{}_LNM] 2026-07-29T08:57:54.765943951Z I0729 08:57:54.765918 3399 base_network_controller_policy.go:1243] Create network policy odh-ai-gateway-infra/maas-api-allow-gateway resources completed, update namespace loglevel 2026-07-29T08:57:54.820488133Z I0729 08:57:54.820443 3399 base_network_controller_policy.go:1190] Adding network policy odh-ai-gateway-infra/maas-api-allow-monitoring for network default 2026-07-29T08:57:54.820545372Z I0729 08:57:54.820497 3399 base_network_controller_policy.go:1085] Policy odh-ai-gateway-infra/maas-api-allow-monitoring added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: redhat-ods-monitoring,},}_LS{}_LNM] 2026-07-29T08:57:54.821884068Z I0729 08:57:54.821856 3399 base_network_controller_policy.go:1243] Create network policy odh-ai-gateway-infra/maas-api-allow-monitoring resources completed, update namespace loglevel 2026-07-29T08:57:54.936160039Z I0729 08:57:54.936098 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j f47212fbc832e549409a567e29744f23a38e92e01af30b5401701e189e3febbf network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:57:54.949844320Z I0729 08:57:54.949673 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-84c7dbbd44-spd7j, hostIfaceName: f47212fbc832e54, network: default, NAD default, SandboxID: "f47212fbc832e549409a567e29744f23a38e92e01af30b5401701e189e3febbf", PCI device ID: , UID: "7fe3cdde-ce75-4e6c-ad5d-7487dac7cf0d", MAC: 0a:58:0a:85:00:35, IPs: [10.133.0.53/23] 2026-07-29T08:57:55.024700909Z I0729 08:57:55.024650 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j f47212fbc832e549409a567e29744f23a38e92e01af30b5401701e189e3febbf network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-84c7dbbd44-spd7j f47212fbc832e549409a567e29744f23a38e92e01af30b5401701e189e3febbf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:4b:b9:94:05:bf\",\"name\":\"f47212fbc832e54\"},{\"mac\":\"0a:58:0a:85:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4f179b6-f198-4094-a658-ff28bf030a7b\"}],\"ips\":[{\"address\":\"10.133.0.53/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:55.102657586Z I0729 08:57:55.102607 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-processing-5cc6cd444b-g5bbx] creating logical port openshift-ingress_payload-processing-5cc6cd444b-g5bbx for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:55.102845725Z I0729 08:57:55.102797 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-processing-5cc6cd444b-g5bbx 2026-07-29T08:57:55.115080387Z I0729 08:57:55.115030 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-processing-5cc6cd444b-g5bbx] addLogicalPort annotation time took 12.278561ms 2026-07-29T08:57:55.116822719Z I0729 08:57:55.116781 3399 pods.go:273] [openshift-ingress/payload-processing-5cc6cd444b-g5bbx] addLogicalPort took 14.204947ms, libovsdb time 1.125438ms 2026-07-29T08:57:55.152410395Z I0729 08:57:55.152368 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9] creating logical port openshift-ingress_payload-pre-processing-687dbb6cf6-6bpz9 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:57:55.152513330Z I0729 08:57:55.152498 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9 2026-07-29T08:57:55.165084170Z I0729 08:57:55.165031 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9] addLogicalPort annotation time took 12.566269ms 2026-07-29T08:57:55.166420415Z I0729 08:57:55.166386 3399 pods.go:273] [openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9] addLogicalPort took 14.038232ms, libovsdb time 863.779µs 2026-07-29T08:57:55.220807365Z I0729 08:57:55.220740 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing for network default 2026-07-29T08:57:55.221659048Z I0729 08:57:55.221609 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T08:57:55.224162695Z I0729 08:57:55.224130 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing resources completed, update namespace loglevel 2026-07-29T08:57:55.472452456Z I0729 08:57:55.472410 3399 cniserver.go:311] [openshift-ingress/payload-processing-5cc6cd444b-g5bbx c590bbe500a536db2d2af7459c1480b1354c23479449f2a61afe568981dc74cf network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:57:55.485310960Z I0729 08:57:55.485251 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-processing-5cc6cd444b-g5bbx, hostIfaceName: c590bbe500a536d, network: default, NAD default, SandboxID: "c590bbe500a536db2d2af7459c1480b1354c23479449f2a61afe568981dc74cf", PCI device ID: , UID: "2695ccb5-8e36-4468-ae15-d2a0c96427a8", MAC: 0a:58:0a:85:00:36, IPs: [10.133.0.54/23] 2026-07-29T08:57:55.525293202Z I0729 08:57:55.525253 3399 cniserver.go:311] [openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9 06a1583d1d6bde20e9836e1306d08091e5d6cea4c196c6b1f253c2f6cc6f67c8 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:57:55.539651046Z I0729 08:57:55.539602 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-pre-processing-687dbb6cf6-6bpz9, hostIfaceName: 06a1583d1d6bde2, network: default, NAD default, SandboxID: "06a1583d1d6bde20e9836e1306d08091e5d6cea4c196c6b1f253c2f6cc6f67c8", PCI device ID: , UID: "1dc48f10-8634-452b-b3c5-b05ddd3d5cbb", MAC: 0a:58:0a:85:00:37, IPs: [10.133.0.55/23] 2026-07-29T08:57:55.554883005Z I0729 08:57:55.554826 3399 cniserver.go:300] [openshift-ingress/payload-processing-5cc6cd444b-g5bbx c590bbe500a536db2d2af7459c1480b1354c23479449f2a61afe568981dc74cf network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-processing-5cc6cd444b-g5bbx c590bbe500a536db2d2af7459c1480b1354c23479449f2a61afe568981dc74cf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:71:44:d0:53:d9\",\"name\":\"c590bbe500a536d\"},{\"mac\":\"0a:58:0a:85:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/62fc4ded-eea5-464b-abd4-d6e1f3f8ec4a\"}],\"ips\":[{\"address\":\"10.133.0.54/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:55.607267868Z I0729 08:57:55.607222 3399 cniserver.go:300] [openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9 06a1583d1d6bde20e9836e1306d08091e5d6cea4c196c6b1f253c2f6cc6f67c8 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-pre-processing-687dbb6cf6-6bpz9 06a1583d1d6bde20e9836e1306d08091e5d6cea4c196c6b1f253c2f6cc6f67c8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:3a:87:db:af:08\",\"name\":\"06a1583d1d6bde2\"},{\"mac\":\"0a:58:0a:85:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46d5455e-675b-44d9-b26e-d29ad18b5b55\"}],\"ips\":[{\"address\":\"10.133.0.55/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:57:58.837461162Z W0729 08:57:58.837413 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api 2026-07-29T08:57:58.851932638Z W0729 08:57:58.851842 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing 2026-07-29T08:57:58.869032838Z W0729 08:57:58.868316 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing 2026-07-29T08:58:02.871444229Z W0729 08:58:02.871369 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:02.871444229Z W0729 08:58:02.871387 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:02.871478945Z W0729 08:58:02.871413 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:02.871478945Z W0729 08:58:02.871433 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:02.871528579Z W0729 08:58:02.871500 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:02.871566793Z W0729 08:58:02.871533 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:04.854677849Z W0729 08:58:04.854630 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api 2026-07-29T08:58:05.772947067Z I0729 08:58:05.769274 3399 namespace.go:89] [llm] adding namespace 2026-07-29T08:58:05.775935419Z I0729 08:58:05.773513 3399 namespace.go:93] [llm] adding namespace took 4.207321ms 2026-07-29T08:58:05.784289114Z I0729 08:58:05.784230 3399 namespace.go:133] [llm] updating namespace 2026-07-29T08:58:05.796252687Z I0729 08:58:05.796211 3399 namespace.go:133] [llm] updating namespace 2026-07-29T08:58:08.505150510Z W0729 08:58:08.505105 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:08.505150510Z W0729 08:58:08.505131 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/maas-default-gateway-openshift-default 2026-07-29T08:58:09.854525075Z W0729 08:58:09.854473 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing 2026-07-29T08:58:09.874450277Z W0729 08:58:09.874408 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-processing 2026-07-29T08:58:30.404676275Z I0729 08:58:30.404633 3399 base_network_controller_pods.go:490] [default/llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl] creating logical port llm_e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T08:58:30.404838992Z I0729 08:58:30.404809 3399 kube.go:234] Patching annotations on pod llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl 2026-07-29T08:58:30.419210316Z I0729 08:58:30.419160 3399 base_network_controller_pods.go:976] [default/llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl] addLogicalPort annotation time took 14.406012ms 2026-07-29T08:58:30.420883190Z I0729 08:58:30.420814 3399 pods.go:273] [llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl] addLogicalPort took 16.199588ms, libovsdb time 1.092613ms 2026-07-29T08:58:30.783837026Z I0729 08:58:30.783795 3399 cniserver.go:311] [llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl bcef7c9f41d7d3ee8e35fedb7782fcbf2b5621b8b307aa78591e52b05bf024ac network default NAD default NAD key default] ADD starting CNI request 2026-07-29T08:58:30.793753926Z I0729 08:58:30.793703 3399 helper_linux.go:533] ConfigureOVS: namespace: llm, podName: e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl, hostIfaceName: bcef7c9f41d7d3e, network: default, NAD default, SandboxID: "bcef7c9f41d7d3ee8e35fedb7782fcbf2b5621b8b307aa78591e52b05bf024ac", PCI device ID: , UID: "d3c42269-67fb-4bd6-99db-6dd56779b097", MAC: 0a:58:0a:85:00:38, IPs: [10.133.0.56/23] 2026-07-29T08:58:30.849266362Z I0729 08:58:30.849217 3399 cniserver.go:300] [llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl bcef7c9f41d7d3ee8e35fedb7782fcbf2b5621b8b307aa78591e52b05bf024ac network default NAD default NAD key default] ADD finished CNI request [llm/e2e-distinct-2-simulated-kserve-67d8d5d6d-r9dpl bcef7c9f41d7d3ee8e35fedb7782fcbf2b5621b8b307aa78591e52b05bf024ac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:cb:fd:a8:a6:fe\",\"name\":\"bcef7c9f41d7d3e\"},{\"mac\":\"0a:58:0a:85:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/61f10389-02c5-40ad-92b1-4cec25a59888\"}],\"ips\":[{\"address\":\"10.133.0.56/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T08:58:33.051774199Z W0729 08:58:33.051729 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/e2e-distinct-2-simulated-kserve-workload-svc 2026-07-29T08:58:40.330246879Z W0729 08:58:40.330152 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/e2e-distinct-simulated-kserve-workload-svc 2026-07-29T08:58:41.112762710Z W0729 08:58:41.112718 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service llm/e2e-distinct-2-simulated-kserve-workload-svc 2026-07-29T08:58:55.336428037Z W0729 08:58:55.336382 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/e2e-trlp-test-simulated-kserve-workload-svc 2026-07-29T08:59:06.399006860Z W0729 08:59:06.398958 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc 2026-07-29T08:59:12.463800096Z W0729 08:59:12.463750 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/facebook-opt-125m-simulated-kserve-workload-svc 2026-07-29T08:59:27.501680667Z W0729 08:59:27.501631 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/premium-simulated-simulated-premium-kserve-workload-svc 2026-07-29T09:00:00.138064419Z I0729 09:00:00.138017 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r] creating logical port odh-ai-gateway-infra_maas-api-key-cleanup-29755260-8jz9r for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:00:00.138190122Z I0729 09:00:00.138171 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:00:00.149047437Z I0729 09:00:00.148997 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r] addLogicalPort annotation time took 10.858631ms 2026-07-29T09:00:00.150549165Z I0729 09:00:00.150514 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r] addLogicalPort took 12.522977ms, libovsdb time 935.235µs 2026-07-29T09:00:00.206378017Z I0729 09:00:00.206333 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:00:00.217836609Z I0729 09:00:00.217786 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-key-cleanup-29755260-8jz9r, hostIfaceName: 39cbcb3fdab7168, network: default, NAD default, SandboxID: "39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd", PCI device ID: , UID: "a067c614-886c-43de-8204-df96f9407736", MAC: 0a:58:0a:85:00:39, IPs: [10.133.0.57/23] 2026-07-29T09:00:00.482388814Z I0729 09:00:00.482327 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:1a:73:dd:f2:00\",\"name\":\"39cbcb3fdab7168\"},{\"mac\":\"0a:58:0a:85:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/95209cd4-d0c4-4226-a4bf-3c2ecb76171e\"}],\"ips\":[{\"address\":\"10.133.0.57/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:00:05.607917684Z I0729 09:00:05.607857 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:00:05.632099359Z E0729 09:00:05.632058 3399 link_network_manager.go:104] Error while syncing link "39cbcb3fdab7168": failed to get address from link "39cbcb3fdab7168": failed to list addresses for "39cbcb3fdab7168": no such device 2026-07-29T09:00:05.683148079Z I0729 09:00:05.683092 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 39cbcb3fdab7168ce1abc11f5680d1d1ebb3f52a7841cb10c2154bb1524dfbfd network default NAD default NAD key default], result "{}", err 2026-07-29T09:00:05.727129226Z I0729 09:00:05.727070 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:00:05.729163507Z I0729 09:00:05.729127 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r, ips: 10.133.0.57 2026-07-29T09:00:05.729197528Z I0729 09:00:05.729180 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:00:05.729465138Z I0729 09:00:05.729445 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:00:05.729674471Z I0729 09:00:05.729643 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:00:05.729856214Z I0729 09:00:05.729839 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:00:05.730022131Z I0729 09:00:05.730004 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:00:05.730190923Z I0729 09:00:05.730175 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:00:29.002065039Z I0729 09:00:29.002009 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 2026-07-29T09:00:35.029348879Z I0729 09:00:35.029301 3399 namespace.go:89] [premium-users-namespace] adding namespace 2026-07-29T09:00:35.030122942Z I0729 09:00:35.030087 3399 namespace.go:93] [premium-users-namespace] adding namespace took 698.267µs 2026-07-29T09:00:35.039457224Z I0729 09:00:35.039418 3399 namespace.go:133] [premium-users-namespace] updating namespace 2026-07-29T09:00:35.047688346Z I0729 09:00:35.047652 3399 namespace.go:133] [premium-users-namespace] updating namespace 2026-07-29T09:00:36.003107578Z I0729 09:00:36.003065 3399 cniserver.go:311] [kuadrant-system/authorino-645fdb8cd6-kjt89 b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:00:36.039530900Z E0729 09:00:36.039492 3399 link_network_manager.go:104] Error while syncing link "b7ea56da285408d": failed to get address from link "b7ea56da285408d": failed to list addresses for "b7ea56da285408d": no such device 2026-07-29T09:00:36.089526053Z I0729 09:00:36.089470 3399 cniserver.go:300] [kuadrant-system/authorino-645fdb8cd6-kjt89 b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa network default NAD default NAD key default] DEL finished CNI request [kuadrant-system/authorino-645fdb8cd6-kjt89 b7ea56da285408de6fc126ebad9d6db08878dc335dca40df230a74f2bd507caa network default NAD default NAD key default], result "{}", err 2026-07-29T09:00:36.131918616Z I0729 09:00:36.131849 3399 pods.go:178] Deleting pod: kuadrant-system/authorino-645fdb8cd6-kjt89 2026-07-29T09:00:36.133446075Z I0729 09:00:36.133401 3399 pods.go:238] Attempting to release IPs for pod: kuadrant-system/authorino-645fdb8cd6-kjt89, ips: 10.133.0.50 2026-07-29T09:00:36.175698940Z I0729 09:00:36.175651 3399 namespace.go:89] [maas-admin] adding namespace 2026-07-29T09:00:36.176346316Z I0729 09:00:36.176317 3399 namespace.go:93] [maas-admin] adding namespace took 641.942µs 2026-07-29T09:00:36.193211063Z I0729 09:00:36.193171 3399 namespace.go:133] [maas-admin] updating namespace 2026-07-29T09:00:36.206498159Z I0729 09:00:36.206459 3399 namespace.go:133] [maas-admin] updating namespace 2026-07-29T09:00:36.769869766Z I0729 09:00:36.769802 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/authorino-645fdb8cd6-kjt89 2026-07-29T09:00:36.769869766Z I0729 09:00:36.769839 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/authorino-645fdb8cd6-kjt89 2026-07-29T09:00:43.001771659Z I0729 09:00:43.001727 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 2026-07-29T09:01:19.018497135Z I0729 09:01:19.018449 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=8 2026-07-29T09:01:22.520367562Z I0729 09:01:22.520321 3399 cniserver.go:311] [opendatahub/maas-controller-95cc89ddf-sdb9x 8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:01:22.545771269Z E0729 09:01:22.545731 3399 link_network_manager.go:104] Error while syncing link "8055bc93172dc47": failed to get address from link "8055bc93172dc47": failed to list addresses for "8055bc93172dc47": no such device 2026-07-29T09:01:22.603279599Z I0729 09:01:22.603224 3399 cniserver.go:300] [opendatahub/maas-controller-95cc89ddf-sdb9x 8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f network default NAD default NAD key default] DEL finished CNI request [opendatahub/maas-controller-95cc89ddf-sdb9x 8055bc93172dc476dbdb3309f2e4cb07fc9837e1d7188455a540a4be6ad87c9f network default NAD default NAD key default], result "{}", err 2026-07-29T09:01:22.646350334Z I0729 09:01:22.646307 3399 pods.go:178] Deleting pod: opendatahub/maas-controller-95cc89ddf-sdb9x 2026-07-29T09:01:22.648267119Z I0729 09:01:22.648219 3399 pods.go:238] Attempting to release IPs for pod: opendatahub/maas-controller-95cc89ddf-sdb9x, ips: 10.133.0.52 2026-07-29T09:01:22.648293837Z I0729 09:01:22.648264 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:01:22.648515910Z I0729 09:01:22.648494 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/maas-controller-allow-monitoring to delete 1 local pods... 2026-07-29T09:01:23.029244616Z I0729 09:01:23.029199 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/maas-controller-95cc89ddf-sdb9x 2026-07-29T09:01:23.029244616Z I0729 09:01:23.029228 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/maas-controller-95cc89ddf-sdb9x 2026-07-29T09:01:23.029244616Z I0729 09:01:23.029239 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/maas-controller-95cc89ddf-sdb9x 2026-07-29T09:01:23.029285262Z I0729 09:01:23.029256 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/maas-controller-95cc89ddf-sdb9x 2026-07-29T09:01:40.031817618Z I0729 09:01:40.031769 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 2026-07-29T09:01:45.900372336Z I0729 09:01:45.900300 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=157 2026-07-29T09:01:50.900973231Z I0729 09:01:50.900932 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=239 2026-07-29T09:02:15.900823977Z I0729 09:02:15.900772 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=27 2026-07-29T09:02:20.026187485Z I0729 09:02:20.026135 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=7 2026-07-29T09:02:23.801796066Z I0729 09:02:23.801751 3399 base_network_controller_pods.go:490] [default/opendatahub/maas-controller-95cc89ddf-r9ktn] creating logical port opendatahub_maas-controller-95cc89ddf-r9ktn for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:02:23.801911543Z I0729 09:02:23.801874 3399 kube.go:234] Patching annotations on pod opendatahub/maas-controller-95cc89ddf-r9ktn 2026-07-29T09:02:23.812844133Z I0729 09:02:23.812739 3399 base_network_controller_pods.go:976] [default/opendatahub/maas-controller-95cc89ddf-r9ktn] addLogicalPort annotation time took 10.906703ms 2026-07-29T09:02:23.814318608Z I0729 09:02:23.814273 3399 pods.go:273] [opendatahub/maas-controller-95cc89ddf-r9ktn] addLogicalPort took 12.534872ms, libovsdb time 1.005769ms 2026-07-29T09:02:24.169359147Z I0729 09:02:24.169309 3399 cniserver.go:311] [opendatahub/maas-controller-95cc89ddf-r9ktn 1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:02:24.179698868Z I0729 09:02:24.179648 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: maas-controller-95cc89ddf-r9ktn, hostIfaceName: 1205eaf1169b045, network: default, NAD default, SandboxID: "1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a", PCI device ID: , UID: "3f7d6458-cbf6-48d2-9415-7550580d1382", MAC: 0a:58:0a:85:00:3a, IPs: [10.133.0.58/23] 2026-07-29T09:02:24.240822996Z I0729 09:02:24.240771 3399 cniserver.go:300] [opendatahub/maas-controller-95cc89ddf-r9ktn 1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a network default NAD default NAD key default] ADD finished CNI request [opendatahub/maas-controller-95cc89ddf-r9ktn 1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:82:60:0e:78:cf\",\"name\":\"1205eaf1169b045\"},{\"mac\":\"0a:58:0a:85:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f01a9d17-0ee5-42f0-ad29-f6bc59f44028\"}],\"ips\":[{\"address\":\"10.133.0.58/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:02:25.377942131Z W0729 09:02:25.377872 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T09:02:25.378085058Z W0729 09:02:25.378065 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T09:02:36.387634193Z W0729 09:02:36.387589 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T09:02:36.391234409Z W0729 09:02:36.391182 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T09:02:41.146477444Z I0729 09:02:41.146430 3399 cniserver.go:311] [opendatahub/maas-controller-95cc89ddf-r9ktn 1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:02:41.169271249Z E0729 09:02:41.169214 3399 link_network_manager.go:104] Error while syncing link "1205eaf1169b045": failed to get address from link "1205eaf1169b045": failed to list addresses for "1205eaf1169b045": no such device 2026-07-29T09:02:41.225979437Z I0729 09:02:41.225930 3399 cniserver.go:300] [opendatahub/maas-controller-95cc89ddf-r9ktn 1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a network default NAD default NAD key default] DEL finished CNI request [opendatahub/maas-controller-95cc89ddf-r9ktn 1205eaf1169b045e30dea7c5f3dddeb4f815bebde761b3417e96b60b706d2b3a network default NAD default NAD key default], result "{}", err 2026-07-29T09:02:41.269210628Z I0729 09:02:41.269163 3399 pods.go:178] Deleting pod: opendatahub/maas-controller-95cc89ddf-r9ktn 2026-07-29T09:02:41.270709867Z I0729 09:02:41.270663 3399 pods.go:238] Attempting to release IPs for pod: opendatahub/maas-controller-95cc89ddf-r9ktn, ips: 10.133.0.58 2026-07-29T09:02:41.270736833Z I0729 09:02:41.270712 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/maas-controller-allow-monitoring to delete 1 local pods... 2026-07-29T09:02:41.270988816Z I0729 09:02:41.270958 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:02:41.470732030Z I0729 09:02:41.470679 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/maas-controller-95cc89ddf-r9ktn 2026-07-29T09:02:41.470732030Z I0729 09:02:41.470712 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/maas-controller-95cc89ddf-r9ktn 2026-07-29T09:02:41.470761848Z I0729 09:02:41.470733 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/maas-controller-95cc89ddf-r9ktn 2026-07-29T09:02:41.470761848Z I0729 09:02:41.470748 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/maas-controller-95cc89ddf-r9ktn 2026-07-29T09:02:42.246442235Z I0729 09:02:42.246396 3399 base_network_controller_pods.go:490] [default/opendatahub/maas-controller-95cc89ddf-7hzfk] creating logical port opendatahub_maas-controller-95cc89ddf-7hzfk for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:02:42.246586363Z I0729 09:02:42.246564 3399 kube.go:234] Patching annotations on pod opendatahub/maas-controller-95cc89ddf-7hzfk 2026-07-29T09:02:42.258436766Z I0729 09:02:42.258383 3399 base_network_controller_pods.go:976] [default/opendatahub/maas-controller-95cc89ddf-7hzfk] addLogicalPort annotation time took 11.829012ms 2026-07-29T09:02:42.259888743Z I0729 09:02:42.259850 3399 pods.go:273] [opendatahub/maas-controller-95cc89ddf-7hzfk] addLogicalPort took 13.470906ms, libovsdb time 812.497µs 2026-07-29T09:02:42.624551109Z I0729 09:02:42.624499 3399 cniserver.go:311] [opendatahub/maas-controller-95cc89ddf-7hzfk 33fcd7c302b3fbda3683ffc7aaeda81aaff5f483a8f8d37ad6ff8d76a24a2cd5 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:02:42.635868214Z I0729 09:02:42.635813 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: maas-controller-95cc89ddf-7hzfk, hostIfaceName: 33fcd7c302b3fbd, network: default, NAD default, SandboxID: "33fcd7c302b3fbda3683ffc7aaeda81aaff5f483a8f8d37ad6ff8d76a24a2cd5", PCI device ID: , UID: "ff3d63b7-84bd-4a02-afe7-e4ca63120277", MAC: 0a:58:0a:85:00:3b, IPs: [10.133.0.59/23] 2026-07-29T09:02:42.691741746Z I0729 09:02:42.691694 3399 cniserver.go:300] [opendatahub/maas-controller-95cc89ddf-7hzfk 33fcd7c302b3fbda3683ffc7aaeda81aaff5f483a8f8d37ad6ff8d76a24a2cd5 network default NAD default NAD key default] ADD finished CNI request [opendatahub/maas-controller-95cc89ddf-7hzfk 33fcd7c302b3fbda3683ffc7aaeda81aaff5f483a8f8d37ad6ff8d76a24a2cd5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:c9:64:54:39:8c\",\"name\":\"33fcd7c302b3fbd\"},{\"mac\":\"0a:58:0a:85:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7887e9c-e20b-4055-9994-16da379405bc\"}],\"ips\":[{\"address\":\"10.133.0.59/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:02:44.494527757Z W0729 09:02:44.494488 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T09:02:44.497489326Z W0729 09:02:44.497453 3399 util.go:881] Failed to build global endpoints for port TCP//9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T09:02:54.042455621Z I0729 09:02:54.042402 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=7 2026-07-29T09:02:55.506835456Z W0729 09:02:55.506792 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service opendatahub/maas-controller-metrics 2026-07-29T09:02:55.509830900Z W0729 09:02:55.509797 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9443: empty IP address endpoints for service opendatahub/maas-controller-webhook-service 2026-07-29T09:03:22.043542135Z I0729 09:03:22.043496 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=9 2026-07-29T09:03:22.742433099Z I0729 09:03:22.742379 3399 namespace.go:89] [e2e-webhook-test-1fd310] adding namespace 2026-07-29T09:03:22.743209771Z I0729 09:03:22.743167 3399 namespace.go:93] [e2e-webhook-test-1fd310] adding namespace took 759.584µs 2026-07-29T09:03:22.761543260Z I0729 09:03:22.761502 3399 namespace.go:133] [e2e-webhook-test-1fd310] updating namespace 2026-07-29T09:03:22.770225715Z I0729 09:03:22.770185 3399 namespace.go:133] [e2e-webhook-test-1fd310] updating namespace 2026-07-29T09:03:22.976366265Z I0729 09:03:22.976301 3399 namespace.go:133] [e2e-webhook-test-1fd310] updating namespace 2026-07-29T09:03:28.980297047Z I0729 09:03:28.980247 3399 namespace.go:133] [e2e-webhook-test-1fd310] updating namespace 2026-07-29T09:03:28.989982544Z I0729 09:03:28.989930 3399 namespace.go:290] [e2e-webhook-test-1fd310] deleting namespace 2026-07-29T09:03:29.093956275Z I0729 09:03:29.093886 3399 namespace.go:89] [e2e-webhook-test-728d2f] adding namespace 2026-07-29T09:03:29.094699599Z I0729 09:03:29.094665 3399 namespace.go:93] [e2e-webhook-test-728d2f] adding namespace took 651.746µs 2026-07-29T09:03:29.116220243Z I0729 09:03:29.116189 3399 namespace.go:133] [e2e-webhook-test-728d2f] updating namespace 2026-07-29T09:03:29.125921598Z I0729 09:03:29.125865 3399 namespace.go:133] [e2e-webhook-test-728d2f] updating namespace 2026-07-29T09:03:29.330398322Z I0729 09:03:29.330348 3399 namespace.go:133] [e2e-webhook-test-728d2f] updating namespace 2026-07-29T09:03:35.118831354Z I0729 09:03:35.118780 3399 namespace.go:133] [e2e-webhook-test-728d2f] updating namespace 2026-07-29T09:03:35.127480553Z I0729 09:03:35.127438 3399 namespace.go:290] [e2e-webhook-test-728d2f] deleting namespace 2026-07-29T09:03:36.901332733Z I0729 09:03:36.901285 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=36 2026-07-29T09:03:58.037624816Z I0729 09:03:58.037577 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 2026-07-29T09:03:59.021934441Z I0729 09:03:59.021875 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=7 2026-07-29T09:04:31.011024729Z I0729 09:04:31.010972 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressIP" totalItems=8 2026-07-29T09:04:53.901294998Z I0729 09:04:53.901247 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=36 2026-07-29T09:05:30.003668391Z I0729 09:05:30.003618 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 2026-07-29T09:06:30.020411959Z I0729 09:06:30.020358 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=6 2026-07-29T09:07:25.900464492Z I0729 09:07:25.900415 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=55 2026-07-29T09:07:34.902653048Z I0729 09:07:34.902609 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=47 2026-07-29T09:07:58.707215793Z I0729 09:07:58.707167 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/kuadrant-operator-controller-manager-86878c769f-29995 2026-07-29T09:07:58.707215793Z I0729 09:07:58.707195 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/kuadrant-operator-controller-manager-86878c769f-29995 2026-07-29T09:08:05.028273995Z I0729 09:08:05.028225 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=6 2026-07-29T09:08:56.034297528Z I0729 09:08:56.034250 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 2026-07-29T09:08:56.097146906Z I0729 09:08:56.097095 3399 base_network_controller_pods.go:490] [default/kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6] creating logical port kuadrant-system_kuadrant-operator-controller-manager-86878c769f-5jrg6 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:08:56.097380350Z I0729 09:08:56.097350 3399 kube.go:234] Patching annotations on pod kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6 2026-07-29T09:08:56.111780760Z I0729 09:08:56.111743 3399 base_network_controller_pods.go:976] [default/kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6] addLogicalPort annotation time took 14.426861ms 2026-07-29T09:08:56.113265966Z I0729 09:08:56.113230 3399 pods.go:273] [kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6] addLogicalPort took 16.157988ms, libovsdb time 839.355µs 2026-07-29T09:08:56.462517046Z I0729 09:08:56.462469 3399 cniserver.go:311] [kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6 a41ee5cad01885941bf3cba4e0403f3f6351be49096a163b0c6506e0a8273731 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:08:56.472724737Z I0729 09:08:56.472682 3399 helper_linux.go:533] ConfigureOVS: namespace: kuadrant-system, podName: kuadrant-operator-controller-manager-86878c769f-5jrg6, hostIfaceName: a41ee5cad018859, network: default, NAD default, SandboxID: "a41ee5cad01885941bf3cba4e0403f3f6351be49096a163b0c6506e0a8273731", PCI device ID: , UID: "3db3f5f5-d8f6-4284-a676-cf3b7e3a7f26", MAC: 0a:58:0a:85:00:3c, IPs: [10.133.0.60/23] 2026-07-29T09:08:56.529533153Z I0729 09:08:56.529486 3399 cniserver.go:300] [kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6 a41ee5cad01885941bf3cba4e0403f3f6351be49096a163b0c6506e0a8273731 network default NAD default NAD key default] ADD finished CNI request [kuadrant-system/kuadrant-operator-controller-manager-86878c769f-5jrg6 a41ee5cad01885941bf3cba4e0403f3f6351be49096a163b0c6506e0a8273731 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:10:85:e7:a2:85\",\"name\":\"a41ee5cad018859\"},{\"mac\":\"0a:58:0a:85:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf45b838-8738-4c03-96e5-96ad7ce99eb0\"}],\"ips\":[{\"address\":\"10.133.0.60/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:08:57.560317209Z W0729 09:08:57.560219 3399 util.go:881] Failed to build global endpoints for port TCP/metrics/8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T09:09:08.572617496Z W0729 09:09:08.572566 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8080: empty IP address endpoints for service kuadrant-system/kuadrant-operator-metrics 2026-07-29T09:09:21.045335933Z I0729 09:09:21.045290 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=7 2026-07-29T09:09:23.004074476Z I0729 09:09:23.004025 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 2026-07-29T09:09:45.903125626Z I0729 09:09:45.903078 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=8 2026-07-29T09:09:59.039817205Z I0729 09:09:59.039768 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 2026-07-29T09:10:38.902615437Z I0729 09:10:38.902562 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=25 2026-07-29T09:11:46.723242087Z I0729 09:11:46.723168 3399 base_network_controller_pods.go:490] [default/llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5] creating logical port llm_e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:11:46.723372561Z I0729 09:11:46.723352 3399 kube.go:234] Patching annotations on pod llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 2026-07-29T09:11:46.749183988Z I0729 09:11:46.749140 3399 base_network_controller_pods.go:976] [default/llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5] addLogicalPort annotation time took 25.830567ms 2026-07-29T09:11:46.750667524Z I0729 09:11:46.750621 3399 pods.go:273] [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5] addLogicalPort took 27.467101ms, libovsdb time 962.917µs 2026-07-29T09:11:47.097595067Z I0729 09:11:47.097509 3399 cniserver.go:311] [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:11:47.112173863Z I0729 09:11:47.112105 3399 helper_linux.go:533] ConfigureOVS: namespace: llm, podName: e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5, hostIfaceName: aaa4e2dfb2ade94, network: default, NAD default, SandboxID: "aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85", PCI device ID: , UID: "ab5c3bd9-96a1-4728-98c2-c02aaec7085a", MAC: 0a:58:0a:85:00:3d, IPs: [10.133.0.61/23] 2026-07-29T09:11:47.183385192Z I0729 09:11:47.183333 3399 cniserver.go:300] [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85 network default NAD default NAD key default] ADD finished CNI request [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:e0:1c:d5:3a:44\",\"name\":\"aaa4e2dfb2ade94\"},{\"mac\":\"0a:58:0a:85:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9e2708ab-17a6-4f9c-8ab0-43a147d7dfc9\"}],\"ips\":[{\"address\":\"10.133.0.61/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:11:47.535534089Z W0729 09:11:47.535483 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/e2e-same-id-a-3dafdacb-kserve-workload-svc 2026-07-29T09:11:55.031705260Z W0729 09:11:55.031657 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service llm/e2e-same-id-b-3dafdacb-kserve-workload-svc 2026-07-29T09:11:57.606029173Z W0729 09:11:57.605979 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service llm/e2e-same-id-a-3dafdacb-kserve-workload-svc 2026-07-29T09:12:02.903461613Z I0729 09:12:02.903413 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=15 2026-07-29T09:12:08.904161358Z I0729 09:12:08.904112 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=10 2026-07-29T09:12:18.044476087Z I0729 09:12:18.044433 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=10 2026-07-29T09:12:41.508433113Z I0729 09:12:41.508383 3399 cniserver.go:311] [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:12:41.544445651Z E0729 09:12:41.544395 3399 link_network_manager.go:104] Error while syncing link "aaa4e2dfb2ade94": failed to get address from link "aaa4e2dfb2ade94": failed to list addresses for "aaa4e2dfb2ade94": no such device 2026-07-29T09:12:41.593918513Z I0729 09:12:41.593846 3399 cniserver.go:300] [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85 network default NAD default NAD key default] DEL finished CNI request [llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 aaa4e2dfb2ade9405d37723a226310cca14fdaba357011178529264d3a6dfc85 network default NAD default NAD key default], result "{}", err 2026-07-29T09:12:41.637619323Z I0729 09:12:41.637576 3399 pods.go:178] Deleting pod: llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 2026-07-29T09:12:41.639396188Z I0729 09:12:41.639352 3399 pods.go:238] Attempting to release IPs for pod: llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5, ips: 10.133.0.61 2026-07-29T09:12:41.847435989Z I0729 09:12:41.847384 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 2026-07-29T09:12:41.847506749Z I0729 09:12:41.847492 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod llm/e2e-same-id-a-3dafdacb-kserve-b68f89f4c-2g6c5 2026-07-29T09:12:43.556423109Z I0729 09:12:43.556371 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod llm/e2e-same-id-b-3dafdacb-kserve-56c68f69c5-qv4bk 2026-07-29T09:12:43.556423109Z I0729 09:12:43.556404 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod llm/e2e-same-id-b-3dafdacb-kserve-56c68f69c5-qv4bk 2026-07-29T09:13:27.902450687Z I0729 09:13:27.902398 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=40 2026-07-29T09:13:40.006125701Z I0729 09:13:40.006077 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 2026-07-29T09:13:47.023586640Z I0729 09:13:47.023529 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=11 2026-07-29T09:13:54.012924136Z I0729 09:13:54.012857 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressIP" totalItems=10 2026-07-29T09:14:46.005795335Z I0729 09:14:46.005752 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 2026-07-29T09:15:04.590729675Z I0729 09:15:04.590668 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:15:04.590729675Z I0729 09:15:04.590699 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:15:04.590729675Z I0729 09:15:04.590710 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:15:04.590729675Z I0729 09:15:04.590722 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:15:04.590779470Z I0729 09:15:04.590732 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:15:04.590779470Z I0729 09:15:04.590744 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:15:06.030565589Z I0729 09:15:06.030514 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=7 2026-07-29T09:15:07.564497008Z W0729 09:15:07.564449 3399 duration.go:464] Config duration recorder: measurement expired for pod/kuadrant-system/0acee64185f523d1d1272e9af2e4d9333e0dcde792ba30e1fa9605b7599j7zt 2026-07-29T09:15:09.022483890Z I0729 09:15:09.022429 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=9 2026-07-29T09:15:16.036302205Z I0729 09:15:16.036244 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 2026-07-29T09:15:47.904815411Z I0729 09:15:47.904762 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=89 2026-07-29T09:15:57.904699743Z I0729 09:15:57.904648 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=14 2026-07-29T09:16:07.524918506Z I0729 09:16:07.524847 3399 namespace.go:89] [e2e-ait-wrong-49fec11f] adding namespace 2026-07-29T09:16:07.525597809Z I0729 09:16:07.525556 3399 namespace.go:93] [e2e-ait-wrong-49fec11f] adding namespace took 677.115µs 2026-07-29T09:16:07.541579073Z I0729 09:16:07.541539 3399 namespace.go:133] [e2e-ait-wrong-49fec11f] updating namespace 2026-07-29T09:16:07.548174703Z I0729 09:16:07.548140 3399 namespace.go:133] [e2e-ait-wrong-49fec11f] updating namespace 2026-07-29T09:16:07.962436143Z I0729 09:16:07.962347 3399 namespace.go:133] [e2e-ait-wrong-49fec11f] updating namespace 2026-07-29T09:16:13.669241577Z I0729 09:16:13.669181 3399 namespace.go:133] [e2e-ait-wrong-49fec11f] updating namespace 2026-07-29T09:16:13.678830599Z I0729 09:16:13.678776 3399 namespace.go:290] [e2e-ait-wrong-49fec11f] deleting namespace 2026-07-29T09:16:13.897802339Z I0729 09:16:13.897756 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:16:14.766863916Z W0729 09:16:14.766818 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ait-94ce66b8-openshift-default 2026-07-29T09:16:14.766863916Z W0729 09:16:14.766842 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ait-94ce66b8-openshift-default 2026-07-29T09:16:16.045197893Z I0729 09:16:16.045152 3399 namespace.go:89] [ai-tenant-e2e-ait-94ce66b8] adding namespace 2026-07-29T09:16:16.045922954Z I0729 09:16:16.045873 3399 namespace.go:93] [ai-tenant-e2e-ait-94ce66b8] adding namespace took 617.489µs 2026-07-29T09:16:16.065842676Z I0729 09:16:16.065802 3399 namespace.go:133] [ai-tenant-e2e-ait-94ce66b8] updating namespace 2026-07-29T09:16:16.073722310Z I0729 09:16:16.073682 3399 namespace.go:133] [ai-tenant-e2e-ait-94ce66b8] updating namespace 2026-07-29T09:16:16.929319505Z I0729 09:16:16.929275 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm] creating logical port openshift-ingress_payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:16:16.929459335Z I0729 09:16:16.929435 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 2026-07-29T09:16:16.944564215Z I0729 09:16:16.944521 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm] addLogicalPort annotation time took 15.12493ms 2026-07-29T09:16:16.946170787Z I0729 09:16:16.946104 3399 pods.go:273] [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm] addLogicalPort took 16.843905ms, libovsdb time 985.242µs 2026-07-29T09:16:17.012420335Z I0729 09:16:17.012382 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-ait-94ce66b8 for network default 2026-07-29T09:16:17.012466582Z I0729 09:16:17.012454 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-ait-94ce66b8 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:16:17.014975751Z I0729 09:16:17.014945 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-ait-94ce66b8 resources completed, update namespace loglevel 2026-07-29T09:16:17.303625439Z I0729 09:16:17.303580 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:16:17.313927710Z I0729 09:16:17.313868 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm, hostIfaceName: 40fd349438ed148, network: default, NAD default, SandboxID: "40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e", PCI device ID: , UID: "7b4f9c42-c818-4a8c-9750-2ad01a24dc90", MAC: 0a:58:0a:85:00:3e, IPs: [10.133.0.62/23] 2026-07-29T09:16:17.377024205Z I0729 09:16:17.376976 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:d8:ed:f2:6e:97\",\"name\":\"40fd349438ed148\"},{\"mac\":\"0a:58:0a:85:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d8ddfe4f-8ddf-4cab-afc0-0541f6872c54\"}],\"ips\":[{\"address\":\"10.133.0.62/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:16:18.042453030Z I0729 09:16:18.042402 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 2026-07-29T09:16:18.114137335Z W0729 09:16:18.114090 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ait-94ce66b8 2026-07-29T09:16:20.799613107Z W0729 09:16:20.799565 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8 2026-07-29T09:16:20.812100761Z W0729 09:16:20.812063 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-ait-94ce66b8 2026-07-29T09:16:29.123718067Z W0729 09:16:29.123671 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ait-94ce66b8 2026-07-29T09:16:31.864476844Z W0729 09:16:31.864390 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ait-94ce66b8-openshift-default 2026-07-29T09:16:31.864476844Z W0729 09:16:31.864419 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ait-94ce66b8-openshift-default 2026-07-29T09:16:44.044142455Z I0729 09:16:44.044089 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:16:44.044142455Z I0729 09:16:44.044133 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:16:44.044190423Z I0729 09:16:44.044146 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:16:44.044190423Z I0729 09:16:44.044157 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:16:44.044190423Z I0729 09:16:44.044170 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:16:44.044190423Z I0729 09:16:44.044180 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:16:46.890276505Z I0729 09:16:46.890238 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-ait-94ce66b8 2026-07-29T09:16:46.890276505Z I0729 09:16:46.890264 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-ait-94ce66b8 2026-07-29T09:16:46.946177199Z I0729 09:16:46.946135 3399 namespace.go:133] [ai-tenant-e2e-ait-94ce66b8] updating namespace 2026-07-29T09:16:46.983538578Z I0729 09:16:46.983491 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983538578Z I0729 09:16:46.983514 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983538578Z I0729 09:16:46.983525 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983538578Z I0729 09:16:46.983533 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983587939Z I0729 09:16:46.983540 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983587939Z I0729 09:16:46.983556 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983587939Z I0729 09:16:46.983565 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:46.983587939Z I0729 09:16:46.983582 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-94ce66b8-8m7td 2026-07-29T09:16:47.000529299Z I0729 09:16:47.000485 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:16:47.024060832Z E0729 09:16:47.024027 3399 link_network_manager.go:104] Error while syncing link "40fd349438ed148": failed to get address from link "40fd349438ed148": failed to list addresses for "40fd349438ed148": no such device 2026-07-29T09:16:47.080261183Z I0729 09:16:47.080221 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 40fd349438ed148a5292b60bbb31592cbe82c2ec026dd9342fefffbe2aa8c22e network default NAD default NAD key default], result "{}", err 2026-07-29T09:16:47.136182878Z I0729 09:16:47.136140 3399 pods.go:178] Deleting pod: openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 2026-07-29T09:16:47.137767666Z I0729 09:16:47.137728 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm, ips: 10.133.0.62 2026-07-29T09:16:47.153808106Z I0729 09:16:47.153771 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:16:47.153940071Z I0729 09:16:47.153913 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:16:47.153940071Z I0729 09:16:47.153928 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:16:47.153940071Z I0729 09:16:47.153936 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:16:47.153980889Z I0729 09:16:47.153945 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:16:47.304579288Z I0729 09:16:47.304531 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 2026-07-29T09:16:47.304579288Z I0729 09:16:47.304568 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-ait-94ce66b8-69648df5cd-tggwm 2026-07-29T09:16:47.356770584Z I0729 09:16:47.356724 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:16:47.579647444Z I0729 09:16:47.579600 3399 namespace.go:89] [ai-tenant-e2e-migrate-610dc5ab] adding namespace 2026-07-29T09:16:47.580277921Z I0729 09:16:47.580245 3399 namespace.go:93] [ai-tenant-e2e-migrate-610dc5ab] adding namespace took 613.362µs 2026-07-29T09:16:47.593483500Z I0729 09:16:47.593442 3399 namespace.go:133] [ai-tenant-e2e-migrate-610dc5ab] updating namespace 2026-07-29T09:16:47.601816344Z I0729 09:16:47.601785 3399 namespace.go:133] [ai-tenant-e2e-migrate-610dc5ab] updating namespace 2026-07-29T09:16:47.792445593Z I0729 09:16:47.792402 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:16:47.933971953Z I0729 09:16:47.933915 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.933971953Z I0729 09:16:47.933944 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.933971953Z I0729 09:16:47.933951 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.933971953Z I0729 09:16:47.933956 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.933971953Z I0729 09:16:47.933964 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.934018347Z I0729 09:16:47.933973 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.934018347Z I0729 09:16:47.933983 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-ait-94ce66b8-77997c8f8d-l5qvr 2026-07-29T09:16:47.946013338Z I0729 09:16:47.945969 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-ait-94ce66b8-65f66bd57b-hrb6g 2026-07-29T09:16:47.946051799Z I0729 09:16:47.946030 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-ait-94ce66b8-65f66bd57b-hrb6g 2026-07-29T09:16:48.957542157Z W0729 09:16:48.957499 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-migrate-610dc5ab-gw-openshift-default 2026-07-29T09:16:48.957542157Z W0729 09:16:48.957522 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-migrate-610dc5ab-gw-openshift-default 2026-07-29T09:16:50.404313824Z I0729 09:16:50.404268 3399 namespace.go:133] [ai-tenant-e2e-migrate-610dc5ab] updating namespace 2026-07-29T09:16:51.316836350Z I0729 09:16:51.316788 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-migrate-610dc5ab for network default 2026-07-29T09:16:51.316952990Z I0729 09:16:51.316931 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-migrate-610dc5ab added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:16:51.318947917Z I0729 09:16:51.318918 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-migrate-610dc5ab resources completed, update namespace loglevel 2026-07-29T09:16:51.996337265Z W0729 09:16:51.996296 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-migrate-610dc5ab 2026-07-29T09:16:52.007637102Z W0729 09:16:52.007601 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-migrate-610dc5ab 2026-07-29T09:16:52.977277738Z W0729 09:16:52.977197 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab 2026-07-29T09:16:52.986344342Z I0729 09:16:52.986289 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-ait-94ce66b8-openshift-default-cd9cbcf46-x5dtr 2026-07-29T09:16:52.986344342Z I0729 09:16:52.986319 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-ait-94ce66b8-openshift-default-cd9cbcf46-x5dtr 2026-07-29T09:17:05.040294480Z W0729 09:17:05.040246 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-migrate-610dc5ab-gw-openshift-default 2026-07-29T09:17:05.040337875Z W0729 09:17:05.040306 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-migrate-610dc5ab-gw-openshift-default 2026-07-29T09:17:15.403202261Z I0729 09:17:15.403151 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:17:15.403354923Z I0729 09:17:15.403329 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:15.418814412Z I0729 09:17:15.418762 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s] addLogicalPort annotation time took 15.482333ms 2026-07-29T09:17:15.420285226Z I0729 09:17:15.420253 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s] addLogicalPort took 17.120719ms, libovsdb time 834.28µs 2026-07-29T09:17:15.771257711Z I0729 09:17:15.771210 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:17:15.784839357Z I0729 09:17:15.784798 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s, hostIfaceName: e7fa72d751ae28f, network: default, NAD default, SandboxID: "e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc", PCI device ID: , UID: "ff8921f2-7740-42da-b890-4dbde89c1451", MAC: 0a:58:0a:85:00:3f, IPs: [10.133.0.63/23] 2026-07-29T09:17:15.842748980Z I0729 09:17:15.842698 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:33:70:0f:42:21\",\"name\":\"e7fa72d751ae28f\"},{\"mac\":\"0a:58:0a:85:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6600f60c-0307-44d6-9853-ad32d9270bef\"}],\"ips\":[{\"address\":\"10.133.0.63/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:17:18.485605771Z I0729 09:17:18.485559 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:17:18.508104266Z E0729 09:17:18.508054 3399 link_network_manager.go:104] Error while syncing link "e7fa72d751ae28f": failed to get address from link "e7fa72d751ae28f": failed to list addresses for "e7fa72d751ae28f": no such device 2026-07-29T09:17:18.509420879Z E0729 09:17:18.509384 3399 link_network_manager.go:104] Error while syncing link "e7fa72d751ae28f": failed to get address from link "e7fa72d751ae28f": failed to list addresses for "e7fa72d751ae28f": no such device 2026-07-29T09:17:18.566094767Z I0729 09:17:18.566040 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s e7fa72d751ae28f88cb949b4095ce8cbf77c37b18b2ab47aef2edc686884fadc network default NAD default NAD key default], result "{}", err 2026-07-29T09:17:18.609385382Z I0729 09:17:18.609338 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:18.611451343Z I0729 09:17:18.611416 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s, ips: 10.133.0.63 2026-07-29T09:17:18.611486000Z I0729 09:17:18.611469 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:17:18.611707454Z I0729 09:17:18.611690 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:17:18.611959440Z I0729 09:17:18.611928 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:17:18.612153191Z I0729 09:17:18.612127 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:17:18.612321156Z I0729 09:17:18.612302 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:17:18.612499171Z I0729 09:17:18.612479 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:17:22.012073259Z I0729 09:17:22.012025 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-migrate-610dc5ab 2026-07-29T09:17:22.012073259Z I0729 09:17:22.012049 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-migrate-610dc5ab 2026-07-29T09:17:22.065820859Z I0729 09:17:22.065778 3399 namespace.go:133] [ai-tenant-e2e-migrate-610dc5ab] updating namespace 2026-07-29T09:17:22.110147613Z I0729 09:17:22.110107 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110203032Z I0729 09:17:22.110191 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110247338Z I0729 09:17:22.110222 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110247338Z I0729 09:17:22.110235 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110247338Z I0729 09:17:22.110243 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110277705Z I0729 09:17:22.110249 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110277705Z I0729 09:17:22.110256 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.110277705Z I0729 09:17:22.110268 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-migrate-610dc5ab-7rx2s 2026-07-29T09:17:22.293539862Z I0729 09:17:22.293494 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:17:22.293539862Z I0729 09:17:22.293525 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:17:22.293580365Z I0729 09:17:22.293540 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:17:22.293580365Z I0729 09:17:22.293551 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:17:22.293580365Z I0729 09:17:22.293562 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:17:22.431449507Z I0729 09:17:22.431400 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:17:22.660541123Z I0729 09:17:22.660499 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:17:23.153593987Z I0729 09:17:23.153547 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-migrate-610dc5ab-776c55f949-5fwcc 2026-07-29T09:17:23.153593987Z I0729 09:17:23.153577 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-migrate-610dc5ab-776c55f949-5fwcc 2026-07-29T09:17:23.172059459Z I0729 09:17:23.172016 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.172134861Z I0729 09:17:23.172108 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.172134861Z I0729 09:17:23.172123 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.172154507Z I0729 09:17:23.172131 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.172154507Z I0729 09:17:23.172138 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.172154507Z I0729 09:17:23.172143 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.172163803Z I0729 09:17:23.172156 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-migrate-610dc5ab-c874bfbf6-jrjvx 2026-07-29T09:17:23.194118919Z I0729 09:17:23.194086 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-migrate-610dc5ab-69949c5b76-rbsbz 2026-07-29T09:17:23.194118919Z I0729 09:17:23.194108 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-migrate-610dc5ab-69949c5b76-rbsbz 2026-07-29T09:17:24.154630667Z W0729 09:17:24.154589 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ait-696b6b1c-openshift-default 2026-07-29T09:17:24.154630667Z W0729 09:17:24.154609 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ait-696b6b1c-openshift-default 2026-07-29T09:17:25.437527262Z I0729 09:17:25.437473 3399 namespace.go:89] [ai-tenant-e2e-ait-696b6b1c] adding namespace 2026-07-29T09:17:25.438313191Z I0729 09:17:25.438280 3399 namespace.go:93] [ai-tenant-e2e-ait-696b6b1c] adding namespace took 781.138µs 2026-07-29T09:17:25.450451676Z I0729 09:17:25.450420 3399 namespace.go:133] [ai-tenant-e2e-ait-696b6b1c] updating namespace 2026-07-29T09:17:25.459431735Z I0729 09:17:25.459399 3399 namespace.go:133] [ai-tenant-e2e-ait-696b6b1c] updating namespace 2026-07-29T09:17:26.424095648Z I0729 09:17:26.424043 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-ait-696b6b1c for network default 2026-07-29T09:17:26.424207565Z I0729 09:17:26.424184 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-ait-696b6b1c added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:17:26.426012897Z I0729 09:17:26.425971 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-ait-696b6b1c resources completed, update namespace loglevel 2026-07-29T09:17:27.167590517Z W0729 09:17:27.167533 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-ait-696b6b1c 2026-07-29T09:17:27.184658762Z W0729 09:17:27.184609 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ait-696b6b1c 2026-07-29T09:17:28.178929360Z W0729 09:17:28.178870 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c 2026-07-29T09:17:28.187635913Z I0729 09:17:28.187588 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-migrate-610dc5ab-gw-openshift-default-6877d98784-xkrfq 2026-07-29T09:17:28.187635913Z I0729 09:17:28.187623 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-migrate-610dc5ab-gw-openshift-default-6877d98784-xkrfq 2026-07-29T09:17:40.245334762Z W0729 09:17:40.245283 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ait-696b6b1c-openshift-default 2026-07-29T09:17:40.245334762Z W0729 09:17:40.245314 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ait-696b6b1c-openshift-default 2026-07-29T09:17:51.202588666Z I0729 09:17:51.202542 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:17:51.202837711Z I0729 09:17:51.202800 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:51.211672907Z I0729 09:17:51.211630 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2] addLogicalPort annotation time took 8.876567ms 2026-07-29T09:17:51.213514034Z I0729 09:17:51.213483 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2] addLogicalPort took 10.961657ms, libovsdb time 1.202841ms 2026-07-29T09:17:51.575531042Z I0729 09:17:51.575481 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:17:51.585538528Z I0729 09:17:51.585491 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2, hostIfaceName: 50a7fdec8db5206, network: default, NAD default, SandboxID: "50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b", PCI device ID: , UID: "aadce72f-a074-4458-897b-0ba0a2471156", MAC: 0a:58:0a:85:00:40, IPs: [10.133.0.64/23] 2026-07-29T09:17:51.646480411Z I0729 09:17:51.646421 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:06:8e:b1:87:09\",\"name\":\"50a7fdec8db5206\"},{\"mac\":\"0a:58:0a:85:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0aa83a03-6501-455c-afdc-b10c817a0ae6\"}],\"ips\":[{\"address\":\"10.133.0.64/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:17:53.700392110Z I0729 09:17:53.700345 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:17:53.724223399Z E0729 09:17:53.724184 3399 link_network_manager.go:104] Error while syncing link "50a7fdec8db5206": failed to get address from link "50a7fdec8db5206": failed to list addresses for "50a7fdec8db5206": no such device 2026-07-29T09:17:53.781723538Z I0729 09:17:53.781669 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 50a7fdec8db520618a8dd896536f72026ca85068103d38c5848db05fbf1cf91b network default NAD default NAD key default], result "{}", err 2026-07-29T09:17:53.823602245Z I0729 09:17:53.823552 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:53.825467266Z I0729 09:17:53.825429 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2, ips: 10.133.0.64 2026-07-29T09:17:53.825494424Z I0729 09:17:53.825482 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:17:53.825780610Z I0729 09:17:53.825754 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:17:53.826004162Z I0729 09:17:53.825982 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:17:53.826216058Z I0729 09:17:53.826198 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:17:53.826394812Z I0729 09:17:53.826370 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:17:53.826623075Z I0729 09:17:53.826601 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:17:56.046841789Z I0729 09:17:56.046795 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=9 2026-07-29T09:17:56.343667186Z I0729 09:17:56.343619 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-ait-696b6b1c 2026-07-29T09:17:56.343667186Z I0729 09:17:56.343641 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-ait-696b6b1c 2026-07-29T09:17:56.407215443Z I0729 09:17:56.407167 3399 namespace.go:133] [ai-tenant-e2e-ait-696b6b1c] updating namespace 2026-07-29T09:17:56.447741447Z I0729 09:17:56.447688 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447741447Z I0729 09:17:56.447716 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447741447Z I0729 09:17:56.447725 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447741447Z I0729 09:17:56.447733 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447797352Z I0729 09:17:56.447743 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447797352Z I0729 09:17:56.447752 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447797352Z I0729 09:17:56.447762 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.447797352Z I0729 09:17:56.447781 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ait-696b6b1c-fxbk2 2026-07-29T09:17:56.469389228Z I0729 09:17:56.469347 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:17:56.469457156Z I0729 09:17:56.469445 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:17:56.469520906Z I0729 09:17:56.469482 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:17:56.469520906Z I0729 09:17:56.469494 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:17:56.469520906Z I0729 09:17:56.469504 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:17:57.332139868Z I0729 09:17:57.332085 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.332197854Z I0729 09:17:57.332173 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.332197854Z I0729 09:17:57.332186 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.332229153Z I0729 09:17:57.332212 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.332229153Z I0729 09:17:57.332222 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.332239688Z I0729 09:17:57.332229 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.332263499Z I0729 09:17:57.332249 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-ait-696b6b1c-646cdbf8d6-b2tbp 2026-07-29T09:17:57.345174738Z I0729 09:17:57.345121 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-ait-696b6b1c-7bb8587fd-8gkg8 2026-07-29T09:17:57.345174738Z I0729 09:17:57.345149 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-ait-696b6b1c-7bb8587fd-8gkg8 2026-07-29T09:17:57.360838322Z I0729 09:17:57.360792 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-ait-696b6b1c-59779c5d6d-tfvwq 2026-07-29T09:17:57.360838322Z I0729 09:17:57.360826 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-ait-696b6b1c-59779c5d6d-tfvwq 2026-07-29T09:17:57.768638764Z I0729 09:17:57.768583 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:17:57.974440280Z I0729 09:17:57.974391 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:17:59.347290830Z W0729 09:17:59.347240 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-derive-3cfa77ca-openshift-default 2026-07-29T09:17:59.347290830Z W0729 09:17:59.347261 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-derive-3cfa77ca-openshift-default 2026-07-29T09:18:00.619689015Z I0729 09:18:00.619631 3399 namespace.go:89] [ai-tenant-e2e-derive-3cfa77ca] adding namespace 2026-07-29T09:18:00.620392515Z I0729 09:18:00.620361 3399 namespace.go:93] [ai-tenant-e2e-derive-3cfa77ca] adding namespace took 700.532µs 2026-07-29T09:18:00.637629402Z I0729 09:18:00.637579 3399 namespace.go:133] [ai-tenant-e2e-derive-3cfa77ca] updating namespace 2026-07-29T09:18:00.648684350Z I0729 09:18:00.648641 3399 namespace.go:133] [ai-tenant-e2e-derive-3cfa77ca] updating namespace 2026-07-29T09:18:01.586365023Z I0729 09:18:01.586319 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-derive-3cfa77ca for network default 2026-07-29T09:18:01.586452119Z I0729 09:18:01.586423 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-derive-3cfa77ca added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:18:01.588390972Z I0729 09:18:01.588353 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-derive-3cfa77ca resources completed, update namespace loglevel 2026-07-29T09:18:02.382099322Z W0729 09:18:02.382053 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-derive-3cfa77ca 2026-07-29T09:18:02.397108199Z W0729 09:18:02.397063 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-derive-3cfa77ca 2026-07-29T09:18:03.378646405Z W0729 09:18:03.378589 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca 2026-07-29T09:18:03.388511326Z I0729 09:18:03.388465 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-ait-696b6b1c-openshift-default-7f5477bb-bvkft 2026-07-29T09:18:03.388511326Z I0729 09:18:03.388492 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-ait-696b6b1c-openshift-default-7f5477bb-bvkft 2026-07-29T09:18:16.441150515Z W0729 09:18:16.441109 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-derive-3cfa77ca-openshift-default 2026-07-29T09:18:16.441150515Z W0729 09:18:16.441135 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-derive-3cfa77ca-openshift-default 2026-07-29T09:18:25.535574035Z I0729 09:18:25.535510 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:18:25.535707638Z I0729 09:18:25.535688 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:25.548412108Z I0729 09:18:25.548099 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8] addLogicalPort annotation time took 12.461343ms 2026-07-29T09:18:25.549738993Z I0729 09:18:25.549706 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8] addLogicalPort took 14.217347ms, libovsdb time 1.036155ms 2026-07-29T09:18:25.901975517Z I0729 09:18:25.901930 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:18:25.912441273Z I0729 09:18:25.912394 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8, hostIfaceName: 93d9eda7e6bc3ef, network: default, NAD default, SandboxID: "93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc", PCI device ID: , UID: "b826fb8c-4f65-4509-bcd2-2bb6ceaa99f3", MAC: 0a:58:0a:85:00:41, IPs: [10.133.0.65/23] 2026-07-29T09:18:25.973610102Z I0729 09:18:25.973560 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:fc:eb:b5:20:ba\",\"name\":\"93d9eda7e6bc3ef\"},{\"mac\":\"0a:58:0a:85:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/221c267b-8f71-46d3-9fcf-6042a5bbd052\"}],\"ips\":[{\"address\":\"10.133.0.65/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:18:27.908441820Z I0729 09:18:27.908396 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:18:27.932016599Z E0729 09:18:27.931974 3399 link_network_manager.go:104] Error while syncing link "93d9eda7e6bc3ef": failed to get address from link "93d9eda7e6bc3ef": failed to list addresses for "93d9eda7e6bc3ef": no such device 2026-07-29T09:18:27.982332083Z I0729 09:18:27.982286 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 93d9eda7e6bc3efc7163b17fd5680375b4d99e5f3685e1f5d601a4ac23dee2fc network default NAD default NAD key default], result "{}", err 2026-07-29T09:18:28.023492619Z I0729 09:18:28.023447 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:28.025557028Z I0729 09:18:28.025530 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8, ips: 10.133.0.65 2026-07-29T09:18:28.025591286Z I0729 09:18:28.025572 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:18:28.025783117Z I0729 09:18:28.025764 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:18:28.025990407Z I0729 09:18:28.025973 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:18:28.026199718Z I0729 09:18:28.026181 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:18:28.026428149Z I0729 09:18:28.026405 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:18:28.026627379Z I0729 09:18:28.026607 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:18:31.608479064Z I0729 09:18:31.608444 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-derive-3cfa77ca 2026-07-29T09:18:31.608479064Z I0729 09:18:31.608465 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-derive-3cfa77ca 2026-07-29T09:18:31.661530415Z I0729 09:18:31.661453 3399 namespace.go:133] [ai-tenant-e2e-derive-3cfa77ca] updating namespace 2026-07-29T09:18:31.700303070Z I0729 09:18:31.700258 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700378849Z I0729 09:18:31.700348 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700378849Z I0729 09:18:31.700359 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700378849Z I0729 09:18:31.700367 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700402226Z I0729 09:18:31.700375 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700402226Z I0729 09:18:31.700383 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700402226Z I0729 09:18:31.700391 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.700415563Z I0729 09:18:31.700409 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-derive-3cfa77ca-vfbs8 2026-07-29T09:18:31.761729413Z I0729 09:18:31.761690 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:18:31.761781253Z I0729 09:18:31.761761 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:18:31.761805957Z I0729 09:18:31.761794 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:18:31.761817789Z I0729 09:18:31.761810 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:18:31.761851453Z I0729 09:18:31.761836 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:18:32.036703449Z I0729 09:18:32.036652 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:18:32.480829989Z I0729 09:18:32.480785 3399 namespace.go:89] [ai-tenant-e2e-mt-a13389cf] adding namespace 2026-07-29T09:18:32.481553540Z I0729 09:18:32.481520 3399 namespace.go:93] [ai-tenant-e2e-mt-a13389cf] adding namespace took 643.298µs 2026-07-29T09:18:32.532836879Z I0729 09:18:32.532790 3399 namespace.go:133] [ai-tenant-e2e-mt-a13389cf] updating namespace 2026-07-29T09:18:32.549273099Z I0729 09:18:32.549221 3399 namespace.go:133] [ai-tenant-e2e-mt-a13389cf] updating namespace 2026-07-29T09:18:32.569544789Z I0729 09:18:32.569495 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-derive-3cfa77ca-59dc8477fb-49rxh 2026-07-29T09:18:32.569544789Z I0729 09:18:32.569531 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-derive-3cfa77ca-59dc8477fb-49rxh 2026-07-29T09:18:32.614734023Z I0729 09:18:32.614672 3399 namespace.go:133] [ai-tenant-e2e-mt-a13389cf] updating namespace 2026-07-29T09:18:32.626673658Z I0729 09:18:32.626622 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-derive-3cfa77ca-566c7bf586-cvgcs 2026-07-29T09:18:32.626673658Z I0729 09:18:32.626650 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-derive-3cfa77ca-566c7bf586-cvgcs 2026-07-29T09:18:32.653295332Z I0729 09:18:32.653242 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:32.653295332Z I0729 09:18:32.653264 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:32.653295332Z I0729 09:18:32.653270 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:32.653295332Z I0729 09:18:32.653275 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:32.653295332Z I0729 09:18:32.653282 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:32.653295332Z I0729 09:18:32.653289 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:32.653331640Z I0729 09:18:32.653301 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-derive-3cfa77ca-cb7bb9cb9-h79jk 2026-07-29T09:18:34.046867012Z I0729 09:18:34.046819 3399 namespace.go:133] [ai-tenant-e2e-mt-a13389cf] updating namespace 2026-07-29T09:18:37.550791124Z I0729 09:18:37.550741 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-derive-3cfa77ca-openshift-default-5d9958476-s8wx6 2026-07-29T09:18:37.550841728Z I0729 09:18:37.550817 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-derive-3cfa77ca-openshift-default-5d9958476-s8wx6 2026-07-29T09:18:39.926434169Z I0729 09:18:39.926384 3399 namespace.go:133] [ai-tenant-e2e-mt-a13389cf] updating namespace 2026-07-29T09:18:39.935940252Z I0729 09:18:39.935882 3399 namespace.go:290] [ai-tenant-e2e-mt-a13389cf] deleting namespace 2026-07-29T09:18:40.036600853Z I0729 09:18:40.036272 3399 namespace.go:89] [ai-tenant-e2e-mt-a7d159e6] adding namespace 2026-07-29T09:18:40.037767584Z I0729 09:18:40.037720 3399 namespace.go:93] [ai-tenant-e2e-mt-a7d159e6] adding namespace took 1.416475ms 2026-07-29T09:18:40.057353136Z I0729 09:18:40.057313 3399 namespace.go:133] [ai-tenant-e2e-mt-a7d159e6] updating namespace 2026-07-29T09:18:40.067713237Z I0729 09:18:40.067672 3399 namespace.go:133] [ai-tenant-e2e-mt-a7d159e6] updating namespace 2026-07-29T09:18:40.148466509Z I0729 09:18:40.148421 3399 namespace.go:133] [ai-tenant-e2e-mt-a7d159e6] updating namespace 2026-07-29T09:18:40.622128415Z I0729 09:18:40.622076 3399 namespace.go:133] [ai-tenant-e2e-mt-a7d159e6] updating namespace 2026-07-29T09:18:44.906942098Z I0729 09:18:44.906880 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=51 2026-07-29T09:18:45.905610337Z I0729 09:18:45.905558 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=53 2026-07-29T09:18:48.008466295Z I0729 09:18:48.008419 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 2026-07-29T09:18:56.904684966Z I0729 09:18:56.904634 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=30 2026-07-29T09:19:04.905524775Z I0729 09:19:04.905483 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=103 2026-07-29T09:19:06.589256424Z I0729 09:19:06.589217 3399 namespace.go:133] [ai-tenant-e2e-mt-a7d159e6] updating namespace 2026-07-29T09:19:12.645011486Z I0729 09:19:12.644950 3399 namespace.go:133] [ai-tenant-e2e-mt-a7d159e6] updating namespace 2026-07-29T09:19:12.654298103Z I0729 09:19:12.654225 3399 namespace.go:290] [ai-tenant-e2e-mt-a7d159e6] deleting namespace 2026-07-29T09:19:12.756945501Z I0729 09:19:12.756868 3399 namespace.go:89] [e2e-unlabeled-08b91a79] adding namespace 2026-07-29T09:19:12.758054084Z I0729 09:19:12.758013 3399 namespace.go:93] [e2e-unlabeled-08b91a79] adding namespace took 1.005376ms 2026-07-29T09:19:12.778599211Z I0729 09:19:12.778553 3399 namespace.go:133] [e2e-unlabeled-08b91a79] updating namespace 2026-07-29T09:19:12.791502476Z I0729 09:19:12.791463 3399 namespace.go:133] [e2e-unlabeled-08b91a79] updating namespace 2026-07-29T09:19:28.692700261Z I0729 09:19:28.692650 3399 namespace.go:133] [e2e-unlabeled-08b91a79] updating namespace 2026-07-29T09:19:34.445693438Z I0729 09:19:34.445644 3399 namespace.go:133] [e2e-unlabeled-08b91a79] updating namespace 2026-07-29T09:19:34.456836509Z I0729 09:19:34.456798 3399 namespace.go:290] [e2e-unlabeled-08b91a79] deleting namespace 2026-07-29T09:19:34.562863014Z I0729 09:19:34.562812 3399 namespace.go:89] [ai-tenant-e2e-mt-00e7eca1] adding namespace 2026-07-29T09:19:34.563547216Z I0729 09:19:34.563501 3399 namespace.go:93] [ai-tenant-e2e-mt-00e7eca1] adding namespace took 640.173µs 2026-07-29T09:19:34.581952728Z I0729 09:19:34.581914 3399 namespace.go:133] [ai-tenant-e2e-mt-00e7eca1] updating namespace 2026-07-29T09:19:34.592575121Z I0729 09:19:34.592535 3399 namespace.go:133] [ai-tenant-e2e-mt-00e7eca1] updating namespace 2026-07-29T09:19:45.148666503Z I0729 09:19:45.148615 3399 namespace.go:133] [ai-tenant-e2e-mt-00e7eca1] updating namespace 2026-07-29T09:19:45.936998392Z I0729 09:19:45.936949 3399 namespace.go:133] [ai-tenant-e2e-mt-00e7eca1] updating namespace 2026-07-29T09:19:52.323954510Z I0729 09:19:52.323880 3399 namespace.go:133] [ai-tenant-e2e-mt-00e7eca1] updating namespace 2026-07-29T09:19:52.334420687Z I0729 09:19:52.334379 3399 namespace.go:290] [ai-tenant-e2e-mt-00e7eca1] deleting namespace 2026-07-29T09:19:52.439962319Z I0729 09:19:52.439873 3399 namespace.go:89] [ai-tenant-e2e-mt-56d5adff] adding namespace 2026-07-29T09:19:52.440640282Z I0729 09:19:52.440595 3399 namespace.go:93] [ai-tenant-e2e-mt-56d5adff] adding namespace took 651.889µs 2026-07-29T09:19:52.471928159Z I0729 09:19:52.471870 3399 namespace.go:133] [ai-tenant-e2e-mt-56d5adff] updating namespace 2026-07-29T09:19:52.486412494Z I0729 09:19:52.486370 3399 namespace.go:133] [ai-tenant-e2e-mt-56d5adff] updating namespace 2026-07-29T09:19:52.565298539Z I0729 09:19:52.565251 3399 namespace.go:133] [ai-tenant-e2e-mt-56d5adff] updating namespace 2026-07-29T09:19:57.683229020Z I0729 09:19:57.683185 3399 namespace.go:89] [ai-tenant-e2e-mt-f39f78a4] adding namespace 2026-07-29T09:19:57.684096334Z I0729 09:19:57.684064 3399 namespace.go:93] [ai-tenant-e2e-mt-f39f78a4] adding namespace took 787.12µs 2026-07-29T09:19:57.697268394Z I0729 09:19:57.697228 3399 namespace.go:133] [ai-tenant-e2e-mt-f39f78a4] updating namespace 2026-07-29T09:19:57.709779585Z I0729 09:19:57.709731 3399 namespace.go:133] [ai-tenant-e2e-mt-f39f78a4] updating namespace 2026-07-29T09:19:57.798610004Z I0729 09:19:57.798562 3399 namespace.go:133] [ai-tenant-e2e-mt-f39f78a4] updating namespace 2026-07-29T09:19:59.587762076Z I0729 09:19:59.587712 3399 namespace.go:133] [ai-tenant-e2e-mt-56d5adff] updating namespace 2026-07-29T09:20:06.334262488Z I0729 09:20:06.334215 3399 namespace.go:133] [ai-tenant-e2e-mt-56d5adff] updating namespace 2026-07-29T09:20:06.367600236Z I0729 09:20:06.367540 3399 namespace.go:290] [ai-tenant-e2e-mt-56d5adff] deleting namespace 2026-07-29T09:20:07.048324648Z I0729 09:20:07.048262 3399 namespace.go:133] [ai-tenant-e2e-mt-f39f78a4] updating namespace 2026-07-29T09:20:12.866626545Z I0729 09:20:12.866581 3399 namespace.go:133] [ai-tenant-e2e-mt-f39f78a4] updating namespace 2026-07-29T09:20:12.882745867Z I0729 09:20:12.882703 3399 namespace.go:290] [ai-tenant-e2e-mt-f39f78a4] deleting namespace 2026-07-29T09:20:13.129172021Z I0729 09:20:13.129110 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:20:14.982345209Z W0729 09:20:14.982294 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-mt-6f2e14d7-openshift-default 2026-07-29T09:20:14.982345209Z W0729 09:20:14.982315 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-mt-6f2e14d7-openshift-default 2026-07-29T09:20:17.057376133Z I0729 09:20:17.057331 3399 namespace.go:89] [ai-tenant-e2e-mt-6f2e14d7] adding namespace 2026-07-29T09:20:17.058094504Z I0729 09:20:17.058056 3399 namespace.go:93] [ai-tenant-e2e-mt-6f2e14d7] adding namespace took 632.791µs 2026-07-29T09:20:17.070374703Z I0729 09:20:17.070334 3399 namespace.go:133] [ai-tenant-e2e-mt-6f2e14d7] updating namespace 2026-07-29T09:20:17.079740988Z I0729 09:20:17.079700 3399 namespace.go:133] [ai-tenant-e2e-mt-6f2e14d7] updating namespace 2026-07-29T09:20:18.170735597Z I0729 09:20:18.170680 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-mt-6f2e14d7 for network default 2026-07-29T09:20:18.170819662Z I0729 09:20:18.170780 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-mt-6f2e14d7 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:20:18.172755667Z I0729 09:20:18.172721 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-mt-6f2e14d7 resources completed, update namespace loglevel 2026-07-29T09:20:19.003912890Z W0729 09:20:19.003851 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-mt-6f2e14d7 2026-07-29T09:20:19.018632090Z W0729 09:20:19.018588 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-mt-6f2e14d7 2026-07-29T09:20:19.034743419Z W0729 09:20:19.034697 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7 2026-07-29T09:20:29.063474457Z W0729 09:20:29.063421 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-mt-6f2e14d7-openshift-default 2026-07-29T09:20:29.063474457Z W0729 09:20:29.063449 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-mt-6f2e14d7-openshift-default 2026-07-29T09:20:42.071825824Z I0729 09:20:42.071775 3399 namespace.go:133] [ai-tenant-e2e-mt-6f2e14d7] updating namespace 2026-07-29T09:20:42.701692894Z I0729 09:20:42.701644 3399 namespace.go:89] [e2e-mt-other-6f2e14d7] adding namespace 2026-07-29T09:20:42.702362415Z I0729 09:20:42.702324 3399 namespace.go:93] [e2e-mt-other-6f2e14d7] adding namespace took 653.892µs 2026-07-29T09:20:42.716139789Z I0729 09:20:42.716104 3399 namespace.go:133] [e2e-mt-other-6f2e14d7] updating namespace 2026-07-29T09:20:42.723019354Z I0729 09:20:42.722973 3399 namespace.go:133] [e2e-mt-other-6f2e14d7] updating namespace 2026-07-29T09:20:43.134669332Z I0729 09:20:43.134627 3399 namespace.go:133] [e2e-mt-other-6f2e14d7] updating namespace 2026-07-29T09:20:48.897774124Z I0729 09:20:48.897720 3399 namespace.go:133] [e2e-mt-other-6f2e14d7] updating namespace 2026-07-29T09:20:48.910796215Z I0729 09:20:48.910765 3399 namespace.go:290] [e2e-mt-other-6f2e14d7] deleting namespace 2026-07-29T09:20:49.288172391Z I0729 09:20:49.288122 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:20:49.288435879Z I0729 09:20:49.288399 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:49.303747133Z I0729 09:20:49.303699 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq] addLogicalPort annotation time took 15.369461ms 2026-07-29T09:20:49.305272193Z I0729 09:20:49.305236 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq] addLogicalPort took 17.136723ms, libovsdb time 895.917µs 2026-07-29T09:20:49.656330549Z I0729 09:20:49.656279 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:20:49.666517530Z I0729 09:20:49.666465 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq, hostIfaceName: d2c72d50fdfc5c4, network: default, NAD default, SandboxID: "d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206", PCI device ID: , UID: "872309a5-2911-4d22-b842-89c3bedd5aba", MAC: 0a:58:0a:85:00:42, IPs: [10.133.0.66/23] 2026-07-29T09:20:49.730217520Z I0729 09:20:49.730163 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206 network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:87:59:16:2c:54\",\"name\":\"d2c72d50fdfc5c4\"},{\"mac\":\"0a:58:0a:85:00:42\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0151a52f-2e83-4f78-851d-c848b1177b7d\"}],\"ips\":[{\"address\":\"10.133.0.66/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:20:51.756603390Z I0729 09:20:51.756558 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:20:51.777121672Z E0729 09:20:51.777084 3399 link_network_manager.go:104] Error while syncing link "d2c72d50fdfc5c4": failed to get address from link "d2c72d50fdfc5c4": failed to list addresses for "d2c72d50fdfc5c4": no such device 2026-07-29T09:20:51.839039576Z I0729 09:20:51.838989 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206 network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq d2c72d50fdfc5c4660655d19b6ba0a2fef101806cb04ef7a3b6e0db40b000206 network default NAD default NAD key default], result "{}", err 2026-07-29T09:20:51.880285747Z I0729 09:20:51.880240 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:51.881966048Z I0729 09:20:51.881934 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq, ips: 10.133.0.66 2026-07-29T09:20:51.881998389Z I0729 09:20:51.881986 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:20:51.882207667Z I0729 09:20:51.882190 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:20:51.882386881Z I0729 09:20:51.882371 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:20:51.882587887Z I0729 09:20:51.882569 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:20:51.882807657Z I0729 09:20:51.882791 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:20:51.882998990Z I0729 09:20:51.882973 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:20:59.817988119Z I0729 09:20:59.817935 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-mt-6f2e14d7 2026-07-29T09:20:59.817988119Z I0729 09:20:59.817962 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-mt-6f2e14d7 2026-07-29T09:20:59.888209683Z I0729 09:20:59.888168 3399 namespace.go:133] [ai-tenant-e2e-mt-6f2e14d7] updating namespace 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936262 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936298 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936306 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936313 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936319 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936326 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936351493Z I0729 09:20:59.936335 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:20:59.936391274Z I0729 09:20:59.936352 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-6f2e14d7-rbchq 2026-07-29T09:21:00.032958630Z I0729 09:21:00.032884 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=7 2026-07-29T09:21:00.053723704Z I0729 09:21:00.053678 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:21:00.053723704Z I0729 09:21:00.053706 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:21:00.053723704Z I0729 09:21:00.053718 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:21:00.053769005Z I0729 09:21:00.053730 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:21:00.053769005Z I0729 09:21:00.053741 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:21:00.258229870Z I0729 09:21:00.258173 3399 namespace.go:133] [ai-tenant-e2e-mt-6f2e14d7] updating namespace 2026-07-29T09:21:00.500570322Z I0729 09:21:00.500524 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.500646960Z I0729 09:21:00.500619 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.500646960Z I0729 09:21:00.500633 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.500646960Z I0729 09:21:00.500640 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.500679163Z I0729 09:21:00.500649 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.500679163Z I0729 09:21:00.500658 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.500679163Z I0729 09:21:00.500673 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-mt-6f2e14d7-5cc9c5b675-c8mv8 2026-07-29T09:21:00.512624665Z I0729 09:21:00.512588 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-mt-6f2e14d7-74b9b8bb8d-k6t67 2026-07-29T09:21:00.512687348Z I0729 09:21:00.512668 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-mt-6f2e14d7-74b9b8bb8d-k6t67 2026-07-29T09:21:00.528325751Z I0729 09:21:00.528294 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-mt-6f2e14d7-7b7ccf786b-mdjjb 2026-07-29T09:21:00.528325751Z I0729 09:21:00.528314 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-mt-6f2e14d7-7b7ccf786b-mdjjb 2026-07-29T09:21:06.649608152Z I0729 09:21:06.649563 3399 namespace.go:133] [ai-tenant-e2e-mt-6f2e14d7] updating namespace 2026-07-29T09:21:06.673039297Z I0729 09:21:06.672995 3399 namespace.go:290] [ai-tenant-e2e-mt-6f2e14d7] deleting namespace 2026-07-29T09:21:07.203249412Z I0729 09:21:07.203201 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:21:07.303108246Z I0729 09:21:07.303056 3399 namespace.go:89] [e2e-webhook-sub-ec73f78e] adding namespace 2026-07-29T09:21:07.303868988Z I0729 09:21:07.303825 3399 namespace.go:93] [e2e-webhook-sub-ec73f78e] adding namespace took 658.606µs 2026-07-29T09:21:07.320416823Z I0729 09:21:07.320384 3399 namespace.go:133] [e2e-webhook-sub-ec73f78e] updating namespace 2026-07-29T09:21:07.343819473Z I0729 09:21:07.343783 3399 namespace.go:133] [e2e-webhook-sub-ec73f78e] updating namespace 2026-07-29T09:21:07.541494539Z I0729 09:21:07.541446 3399 namespace.go:133] [e2e-webhook-sub-ec73f78e] updating namespace 2026-07-29T09:21:13.266743217Z I0729 09:21:13.266681 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-mt-6f2e14d7-openshift-default-5766696dc6-ccrml 2026-07-29T09:21:13.266743217Z I0729 09:21:13.266712 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-mt-6f2e14d7-openshift-default-5766696dc6-ccrml 2026-07-29T09:21:13.758074596Z I0729 09:21:13.758026 3399 namespace.go:133] [e2e-webhook-sub-ec73f78e] updating namespace 2026-07-29T09:21:13.768222282Z I0729 09:21:13.768153 3399 namespace.go:290] [e2e-webhook-sub-ec73f78e] deleting namespace 2026-07-29T09:21:13.875391247Z I0729 09:21:13.875341 3399 namespace.go:89] [e2e-webhook-auth-3bf4b8c3] adding namespace 2026-07-29T09:21:13.876215448Z I0729 09:21:13.876182 3399 namespace.go:93] [e2e-webhook-auth-3bf4b8c3] adding namespace took 742.221µs 2026-07-29T09:21:13.894715009Z I0729 09:21:13.894679 3399 namespace.go:133] [e2e-webhook-auth-3bf4b8c3] updating namespace 2026-07-29T09:21:13.922551764Z I0729 09:21:13.922511 3399 namespace.go:133] [e2e-webhook-auth-3bf4b8c3] updating namespace 2026-07-29T09:21:14.124393217Z I0729 09:21:14.124345 3399 namespace.go:133] [e2e-webhook-auth-3bf4b8c3] updating namespace 2026-07-29T09:21:20.098311994Z I0729 09:21:20.098262 3399 namespace.go:133] [e2e-webhook-auth-3bf4b8c3] updating namespace 2026-07-29T09:21:20.110402936Z I0729 09:21:20.110356 3399 namespace.go:290] [e2e-webhook-auth-3bf4b8c3] deleting namespace 2026-07-29T09:21:26.934468943Z I0729 09:21:26.934421 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:21:28.340105675Z W0729 09:21:28.340060 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-mt-2dc45aaa-openshift-default 2026-07-29T09:21:28.340105675Z W0729 09:21:28.340086 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-mt-2dc45aaa-openshift-default 2026-07-29T09:21:30.147269988Z I0729 09:21:30.147224 3399 namespace.go:89] [ai-tenant-e2e-mt-2dc45aaa] adding namespace 2026-07-29T09:21:30.148045523Z I0729 09:21:30.148010 3399 namespace.go:93] [ai-tenant-e2e-mt-2dc45aaa] adding namespace took 661.006µs 2026-07-29T09:21:30.164998756Z I0729 09:21:30.164942 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:21:30.175680573Z I0729 09:21:30.175647 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:21:31.226109305Z I0729 09:21:31.226063 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-mt-2dc45aaa for network default 2026-07-29T09:21:31.226273954Z I0729 09:21:31.226238 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-mt-2dc45aaa added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:21:31.228235307Z I0729 09:21:31.228211 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-mt-2dc45aaa resources completed, update namespace loglevel 2026-07-29T09:21:32.357340120Z W0729 09:21:32.357279 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa 2026-07-29T09:21:32.372646762Z W0729 09:21:32.372603 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-mt-2dc45aaa 2026-07-29T09:21:32.386651499Z W0729 09:21:32.386610 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-mt-2dc45aaa 2026-07-29T09:21:41.007573867Z I0729 09:21:41.007528 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 2026-07-29T09:21:45.428468542Z W0729 09:21:45.428423 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-mt-2dc45aaa-openshift-default 2026-07-29T09:21:45.428468542Z W0729 09:21:45.428448 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-mt-2dc45aaa-openshift-default 2026-07-29T09:21:52.046323470Z I0729 09:21:52.046273 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=10 2026-07-29T09:21:55.127499940Z I0729 09:21:55.127455 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:21:59.208743198Z I0729 09:21:59.208700 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk] creating logical port ai-tenant-e2e-mt-2dc45aaa_e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:21:59.208963576Z I0729 09:21:59.208929 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 2026-07-29T09:21:59.222466174Z I0729 09:21:59.222415 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk] addLogicalPort annotation time took 13.565837ms 2026-07-29T09:21:59.224340011Z I0729 09:21:59.224302 3399 pods.go:273] [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk] addLogicalPort took 15.620751ms, libovsdb time 1.182267ms 2026-07-29T09:21:59.577149184Z I0729 09:21:59.577104 3399 cniserver.go:311] [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:21:59.587871142Z I0729 09:21:59.587818 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-mt-2dc45aaa, podName: e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk, hostIfaceName: 84daee643ec962c, network: default, NAD default, SandboxID: "84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e", PCI device ID: , UID: "82197a59-00dd-4423-8f3d-97ca95029013", MAC: 0a:58:0a:85:00:43, IPs: [10.133.0.67/23] 2026-07-29T09:21:59.645018437Z I0729 09:21:59.644965 3399 cniserver.go:300] [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:e2:f4:c7:d3:1b\",\"name\":\"84daee643ec962c\"},{\"mac\":\"0a:58:0a:85:00:43\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a083c29d-691d-4a54-8adc-49b9f52abde4\"}],\"ips\":[{\"address\":\"10.133.0.67/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:22:00.145246443Z W0729 09:22:00.145203 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-workload-svc 2026-07-29T09:22:10.217769290Z W0729 09:22:10.217721 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-workload-svc 2026-07-29T09:22:26.757685815Z I0729 09:22:26.757640 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:22:26.757763579Z I0729 09:22:26.757751 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:22:26.757818043Z I0729 09:22:26.757788 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:22:26.757818043Z I0729 09:22:26.757806 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:22:26.757852075Z I0729 09:22:26.757819 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:22:26.757852075Z I0729 09:22:26.757835 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:22:29.014999364Z I0729 09:22:29.014955 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressIP" totalItems=9 2026-07-29T09:22:39.850423262Z I0729 09:22:39.850386 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-mt-2dc45aaa 2026-07-29T09:22:39.850451162Z I0729 09:22:39.850422 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-mt-2dc45aaa 2026-07-29T09:22:39.914773181Z I0729 09:22:39.914702 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:22:39.962015503Z I0729 09:22:39.961974 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962093024Z I0729 09:22:39.962059 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962093024Z I0729 09:22:39.962073 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962093024Z I0729 09:22:39.962081 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962093024Z I0729 09:22:39.962086 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962119526Z I0729 09:22:39.962091 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962119526Z I0729 09:22:39.962100 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:39.962119526Z I0729 09:22:39.962115 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-mt-2dc45aaa-wvts2 2026-07-29T09:22:40.095692769Z I0729 09:22:40.095646 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:22:40.095692769Z I0729 09:22:40.095681 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:22:40.095733243Z I0729 09:22:40.095694 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:22:40.095733243Z I0729 09:22:40.095719 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:22:40.095745322Z I0729 09:22:40.095737 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:22:40.701927388Z I0729 09:22:40.701854 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.701927388Z I0729 09:22:40.701881 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.701927388Z I0729 09:22:40.701916 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.701978194Z I0729 09:22:40.701924 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.701978194Z I0729 09:22:40.701933 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.701978194Z I0729 09:22:40.701944 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.701978194Z I0729 09:22:40.701962 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-mt-2dc45aaa-76c9fc4b8c-cn7x9 2026-07-29T09:22:40.717411242Z I0729 09:22:40.717365 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-mt-2dc45aaa-858b85985d-fgr2z 2026-07-29T09:22:40.717450415Z I0729 09:22:40.717431 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-mt-2dc45aaa-858b85985d-fgr2z 2026-07-29T09:22:40.731790074Z I0729 09:22:40.731749 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-mt-2dc45aaa-574f77888d-4czhb 2026-07-29T09:22:40.731827996Z I0729 09:22:40.731803 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-mt-2dc45aaa-574f77888d-4czhb 2026-07-29T09:22:42.025695478Z I0729 09:22:42.025649 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=10 2026-07-29T09:22:43.019569792Z I0729 09:22:43.019511 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:22:48.061852325Z I0729 09:22:48.061811 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:23:03.226978081Z I0729 09:23:03.226933 3399 cniserver.go:311] [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:23:03.263947759Z E0729 09:23:03.263909 3399 link_network_manager.go:104] Error while syncing link "84daee643ec962c": failed to get address from link "84daee643ec962c": failed to list addresses for "84daee643ec962c": no such device 2026-07-29T09:23:03.310219837Z I0729 09:23:03.310168 3399 cniserver.go:300] [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 84daee643ec962c1d3ac78b822e7566380ec4fbe5ee6975f93c6dcf99d11414e network default NAD default NAD key default], result "{}", err 2026-07-29T09:23:03.353864407Z I0729 09:23:03.353813 3399 pods.go:178] Deleting pod: ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 2026-07-29T09:23:03.355558989Z I0729 09:23:03.355525 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk, ips: 10.133.0.67 2026-07-29T09:23:03.528378132Z I0729 09:23:03.528321 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 2026-07-29T09:23:03.528378132Z I0729 09:23:03.528357 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 2026-07-29T09:23:25.191383253Z I0729 09:23:25.191324 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:23:30.679402738Z I0729 09:23:30.679340 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:23:45.027267369Z I0729 09:23:45.027215 3399 namespace.go:133] [ai-tenant-e2e-mt-2dc45aaa] updating namespace 2026-07-29T09:23:45.037447625Z I0729 09:23:45.037412 3399 namespace.go:290] [ai-tenant-e2e-mt-2dc45aaa] deleting namespace 2026-07-29T09:23:45.567255004Z I0729 09:23:45.567205 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:23:46.651231079Z I0729 09:23:46.651187 3399 namespace.go:89] [ai-tenant-e2e-mt-b73c2c2b] adding namespace 2026-07-29T09:23:46.651948124Z I0729 09:23:46.651885 3399 namespace.go:93] [ai-tenant-e2e-mt-b73c2c2b] adding namespace took 674.576µs 2026-07-29T09:23:46.668517945Z I0729 09:23:46.668477 3399 namespace.go:133] [ai-tenant-e2e-mt-b73c2c2b] updating namespace 2026-07-29T09:23:46.678422413Z I0729 09:23:46.678377 3399 namespace.go:133] [ai-tenant-e2e-mt-b73c2c2b] updating namespace 2026-07-29T09:23:46.770935850Z I0729 09:23:46.770855 3399 namespace.go:133] [ai-tenant-e2e-mt-b73c2c2b] updating namespace 2026-07-29T09:23:47.526155298Z I0729 09:23:47.526102 3399 namespace.go:89] [ai-tenant-e2e-mt-7f6533c0] adding namespace 2026-07-29T09:23:47.526950486Z I0729 09:23:47.526884 3399 namespace.go:93] [ai-tenant-e2e-mt-7f6533c0] adding namespace took 731.377µs 2026-07-29T09:23:47.547143782Z I0729 09:23:47.547103 3399 namespace.go:133] [ai-tenant-e2e-mt-7f6533c0] updating namespace 2026-07-29T09:23:47.601551232Z I0729 09:23:47.601501 3399 namespace.go:133] [ai-tenant-e2e-mt-7f6533c0] updating namespace 2026-07-29T09:23:47.639679923Z I0729 09:23:47.639630 3399 namespace.go:133] [ai-tenant-e2e-mt-7f6533c0] updating namespace 2026-07-29T09:23:49.024481955Z I0729 09:23:49.024429 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=9 2026-07-29T09:23:49.085628104Z I0729 09:23:49.085578 3399 namespace.go:133] [ai-tenant-e2e-mt-b73c2c2b] updating namespace 2026-07-29T09:23:51.003961355Z I0729 09:23:51.003887 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-mt-2dc45aaa-openshift-default-7779bb56dd-tf855 2026-07-29T09:23:51.003961355Z I0729 09:23:51.003941 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-mt-2dc45aaa-openshift-default-7779bb56dd-tf855 2026-07-29T09:23:55.470103853Z I0729 09:23:55.470060 3399 namespace.go:133] [ai-tenant-e2e-mt-b73c2c2b] updating namespace 2026-07-29T09:23:55.490207734Z I0729 09:23:55.490172 3399 namespace.go:290] [ai-tenant-e2e-mt-b73c2c2b] deleting namespace 2026-07-29T09:23:56.129967376Z I0729 09:23:56.129916 3399 namespace.go:133] [ai-tenant-e2e-mt-7f6533c0] updating namespace 2026-07-29T09:24:03.144242129Z I0729 09:24:03.144193 3399 namespace.go:133] [ai-tenant-e2e-mt-7f6533c0] updating namespace 2026-07-29T09:24:03.155305994Z I0729 09:24:03.155265 3399 namespace.go:290] [ai-tenant-e2e-mt-7f6533c0] deleting namespace 2026-07-29T09:24:03.260605226Z I0729 09:24:03.260555 3399 namespace.go:89] [ai-tenant-e2e-mt-29a26107] adding namespace 2026-07-29T09:24:03.261260356Z I0729 09:24:03.261206 3399 namespace.go:93] [ai-tenant-e2e-mt-29a26107] adding namespace took 625.339µs 2026-07-29T09:24:03.276614924Z I0729 09:24:03.276564 3399 namespace.go:133] [ai-tenant-e2e-mt-29a26107] updating namespace 2026-07-29T09:24:03.285416176Z I0729 09:24:03.285385 3399 namespace.go:133] [ai-tenant-e2e-mt-29a26107] updating namespace 2026-07-29T09:24:13.835487176Z I0729 09:24:13.835381 3399 namespace.go:133] [ai-tenant-e2e-mt-29a26107] updating namespace 2026-07-29T09:24:14.171376503Z I0729 09:24:14.171329 3399 namespace.go:133] [ai-tenant-e2e-mt-29a26107] updating namespace 2026-07-29T09:24:23.038441706Z I0729 09:24:23.038396 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 2026-07-29T09:24:29.907126402Z I0729 09:24:29.907072 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=425 2026-07-29T09:24:33.045259566Z I0729 09:24:33.045213 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 2026-07-29T09:24:35.125783452Z I0729 09:24:35.125736 3399 namespace.go:133] [ai-tenant-e2e-mt-29a26107] updating namespace 2026-07-29T09:24:41.481505935Z I0729 09:24:41.481461 3399 namespace.go:133] [ai-tenant-e2e-mt-29a26107] updating namespace 2026-07-29T09:24:41.490453316Z I0729 09:24:41.490246 3399 namespace.go:290] [ai-tenant-e2e-mt-29a26107] deleting namespace 2026-07-29T09:24:41.820385391Z I0729 09:24:41.820097 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:24:43.226718710Z W0729 09:24:43.226667 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-inf-a-306d65-openshift-default 2026-07-29T09:24:43.226718710Z W0729 09:24:43.226695 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-inf-a-306d65-openshift-default 2026-07-29T09:24:45.069032765Z I0729 09:24:45.068986 3399 namespace.go:89] [ai-tenant-e2e-inf-a-306d65] adding namespace 2026-07-29T09:24:45.069813419Z I0729 09:24:45.069780 3399 namespace.go:93] [ai-tenant-e2e-inf-a-306d65] adding namespace took 702.883µs 2026-07-29T09:24:45.081954313Z I0729 09:24:45.081917 3399 namespace.go:133] [ai-tenant-e2e-inf-a-306d65] updating namespace 2026-07-29T09:24:45.093975978Z I0729 09:24:45.093937 3399 namespace.go:133] [ai-tenant-e2e-inf-a-306d65] updating namespace 2026-07-29T09:24:46.143948687Z I0729 09:24:46.143904 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-inf-a-306d65 for network default 2026-07-29T09:24:46.144069699Z I0729 09:24:46.144044 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-inf-a-306d65 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:24:46.146017186Z I0729 09:24:46.145969 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-inf-a-306d65 resources completed, update namespace loglevel 2026-07-29T09:24:47.251992495Z W0729 09:24:47.251944 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-inf-a-306d65 2026-07-29T09:24:47.266246059Z W0729 09:24:47.266204 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-inf-a-306d65 2026-07-29T09:24:47.282461973Z W0729 09:24:47.282418 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65 2026-07-29T09:24:55.907341062Z I0729 09:24:55.907292 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=139 2026-07-29T09:24:59.318927777Z W0729 09:24:59.318861 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-inf-a-306d65-openshift-default 2026-07-29T09:24:59.318927777Z W0729 09:24:59.318885 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-inf-a-306d65-openshift-default 2026-07-29T09:25:00.906799241Z I0729 09:25:00.906763 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=17 2026-07-29T09:25:02.049447939Z I0729 09:25:02.049406 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=9 2026-07-29T09:25:10.051956545Z I0729 09:25:10.051910 3399 namespace.go:133] [ai-tenant-e2e-inf-a-306d65] updating namespace 2026-07-29T09:25:10.402531799Z I0729 09:25:10.402472 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:25:11.392668191Z W0729 09:25:11.392627 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-inf-b-612453-openshift-default 2026-07-29T09:25:11.392668191Z W0729 09:25:11.392648 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-inf-b-612453-openshift-default 2026-07-29T09:25:13.233796902Z I0729 09:25:13.233750 3399 namespace.go:89] [ai-tenant-e2e-inf-b-612453] adding namespace 2026-07-29T09:25:13.234583702Z I0729 09:25:13.234550 3399 namespace.go:93] [ai-tenant-e2e-inf-b-612453] adding namespace took 695.702µs 2026-07-29T09:25:13.250319571Z I0729 09:25:13.250280 3399 namespace.go:133] [ai-tenant-e2e-inf-b-612453] updating namespace 2026-07-29T09:25:13.264823046Z I0729 09:25:13.264781 3399 namespace.go:133] [ai-tenant-e2e-inf-b-612453] updating namespace 2026-07-29T09:25:14.362770419Z I0729 09:25:14.362717 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt] creating logical port openshift-ingress_payload-processing-e2e-inf-b-612453-767cf94c94-q8djt for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:25:14.362958237Z I0729 09:25:14.362933 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 2026-07-29T09:25:14.385087093Z I0729 09:25:14.385032 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt] addLogicalPort annotation time took 22.175715ms 2026-07-29T09:25:14.386741266Z I0729 09:25:14.386701 3399 pods.go:273] [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt] addLogicalPort took 24.022299ms, libovsdb time 1.12963ms 2026-07-29T09:25:14.461639793Z I0729 09:25:14.461594 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-inf-b-612453 for network default 2026-07-29T09:25:14.461800248Z I0729 09:25:14.461758 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-inf-b-612453 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:25:14.464721218Z I0729 09:25:14.464691 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-inf-b-612453 resources completed, update namespace loglevel 2026-07-29T09:25:14.731190559Z I0729 09:25:14.731146 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:25:14.741518501Z I0729 09:25:14.741474 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-processing-e2e-inf-b-612453-767cf94c94-q8djt, hostIfaceName: 297fc57d62fad6f, network: default, NAD default, SandboxID: "297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a", PCI device ID: , UID: "4e936cc3-b547-4b00-a8ef-40f79789b025", MAC: 0a:58:0a:85:00:44, IPs: [10.133.0.68/23] 2026-07-29T09:25:14.801046392Z I0729 09:25:14.800999 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:bb:92:1d:94:2e\",\"name\":\"297fc57d62fad6f\"},{\"mac\":\"0a:58:0a:85:00:44\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8a436aaf-0da9-45c1-984f-4be7b38467b7\"}],\"ips\":[{\"address\":\"10.133.0.68/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:25:15.322800939Z W0729 09:25:15.322756 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-inf-b-612453 2026-07-29T09:25:15.412562677Z W0729 09:25:15.412513 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-inf-b-612453 2026-07-29T09:25:15.432423780Z W0729 09:25:15.432378 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-inf-b-612453 2026-07-29T09:25:26.327556017Z W0729 09:25:26.327497 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-inf-b-612453 2026-07-29T09:25:28.472783805Z W0729 09:25:28.472737 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-inf-b-612453-openshift-default 2026-07-29T09:25:28.472783805Z W0729 09:25:28.472762 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-inf-b-612453-openshift-default 2026-07-29T09:25:38.275808090Z I0729 09:25:38.275744 3399 namespace.go:133] [ai-tenant-e2e-inf-b-612453] updating namespace 2026-07-29T09:25:43.951221825Z W0729 09:25:43.951172 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-inf-a-306d65/test-model-306d65-kserve-workload-svc 2026-07-29T09:26:10.913168660Z I0729 09:26:10.913123 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6] creating logical port ai-tenant-e2e-inf-b-612453_test-model-612453-kserve-686548c7d7-tmlc6 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:26:10.913304043Z I0729 09:26:10.913288 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 2026-07-29T09:26:10.923657494Z I0729 09:26:10.923613 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6] addLogicalPort annotation time took 10.369541ms 2026-07-29T09:26:10.925447600Z I0729 09:26:10.925411 3399 pods.go:273] [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6] addLogicalPort took 12.293216ms, libovsdb time 1.032125ms 2026-07-29T09:26:11.280674529Z I0729 09:26:11.280630 3399 cniserver.go:311] [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:26:11.291249701Z I0729 09:26:11.291211 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-inf-b-612453, podName: test-model-612453-kserve-686548c7d7-tmlc6, hostIfaceName: 3694bebb3980598, network: default, NAD default, SandboxID: "3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092", PCI device ID: , UID: "4eb21969-b9f4-4480-9940-94b9356b997d", MAC: 0a:58:0a:85:00:45, IPs: [10.133.0.69/23] 2026-07-29T09:26:11.350183138Z I0729 09:26:11.350135 3399 cniserver.go:300] [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092 network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:9a:e5:1b:d3:9e\",\"name\":\"3694bebb3980598\"},{\"mac\":\"0a:58:0a:85:00:45\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a3512906-5aca-429b-a660-78b47deb7e43\"}],\"ips\":[{\"address\":\"10.133.0.69/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:26:11.646234216Z W0729 09:26:11.646185 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-workload-svc 2026-07-29T09:26:21.721263209Z W0729 09:26:21.721215 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-workload-svc 2026-07-29T09:26:29.908461031Z I0729 09:26:29.908412 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=112 2026-07-29T09:26:57.012091535Z I0729 09:26:57.012040 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:26:57.012091535Z I0729 09:26:57.012069 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:26:57.012091535Z I0729 09:26:57.012081 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:26:57.012134774Z I0729 09:26:57.012091 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:26:57.012134774Z I0729 09:26:57.012104 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:26:57.012134774Z I0729 09:26:57.012119 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:26:58.981299376Z I0729 09:26:58.981260 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-inf-a-306d65 2026-07-29T09:26:58.981299376Z I0729 09:26:58.981283 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-inf-a-306d65 2026-07-29T09:26:59.040591080Z I0729 09:26:59.040547 3399 namespace.go:133] [ai-tenant-e2e-inf-a-306d65] updating namespace 2026-07-29T09:26:59.097543741Z I0729 09:26:59.097494 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097635014Z I0729 09:26:59.097605 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097635014Z I0729 09:26:59.097618 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097635014Z I0729 09:26:59.097624 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097635014Z I0729 09:26:59.097630 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097658423Z I0729 09:26:59.097637 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097658423Z I0729 09:26:59.097644 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.097669166Z I0729 09:26:59.097657 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-a-306d65-qhbg4 2026-07-29T09:26:59.255748554Z I0729 09:26:59.255705 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:26:59.255748554Z I0729 09:26:59.255729 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:26:59.255748554Z I0729 09:26:59.255738 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:26:59.255789653Z I0729 09:26:59.255745 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:26:59.255789653Z I0729 09:26:59.255757 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:26:59.427670067Z I0729 09:26:59.427615 3399 namespace.go:133] [ai-tenant-e2e-inf-a-306d65] updating namespace 2026-07-29T09:26:59.908433187Z I0729 09:26:59.908387 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-inf-a-306d65-869fbf4cd4-rtzs5 2026-07-29T09:26:59.908433187Z I0729 09:26:59.908423 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-inf-a-306d65-869fbf4cd4-rtzs5 2026-07-29T09:26:59.924643806Z I0729 09:26:59.924602 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.924643806Z I0729 09:26:59.924622 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.924643806Z I0729 09:26:59.924628 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.924643806Z I0729 09:26:59.924633 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.924683257Z I0729 09:26:59.924641 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.924683257Z I0729 09:26:59.924650 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.924683257Z I0729 09:26:59.924665 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-inf-a-306d65-78f95654bb-r4rqq 2026-07-29T09:26:59.950125613Z I0729 09:26:59.950080 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-inf-a-306d65-575757bbcd-7jp2g 2026-07-29T09:26:59.950125613Z I0729 09:26:59.950119 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-inf-a-306d65-575757bbcd-7jp2g 2026-07-29T09:27:04.455021324Z I0729 09:27:04.454973 3399 namespace.go:133] [ai-tenant-e2e-inf-a-306d65] updating namespace 2026-07-29T09:27:08.358730594Z I0729 09:27:08.358683 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-inf-a-306d65/test-model-306d65-kserve-7dffffcd85-p2tdx 2026-07-29T09:27:08.358777750Z I0729 09:27:08.358765 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-inf-a-306d65/test-model-306d65-kserve-7dffffcd85-p2tdx 2026-07-29T09:27:08.938364033Z I0729 09:27:08.938321 3399 cniserver.go:311] [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:27:08.974785554Z E0729 09:27:08.974746 3399 link_network_manager.go:104] Error while syncing link "3694bebb3980598": failed to get address from link "3694bebb3980598": failed to list addresses for "3694bebb3980598": no such device 2026-07-29T09:27:09.017869656Z I0729 09:27:09.017825 3399 cniserver.go:300] [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092 network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 3694bebb3980598b80b44d2bb58e5987273a0ee3cf83b2d582ae4e330be86092 network default NAD default NAD key default], result "{}", err 2026-07-29T09:27:09.061729336Z I0729 09:27:09.061684 3399 pods.go:178] Deleting pod: ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 2026-07-29T09:27:09.063574764Z I0729 09:27:09.063525 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6, ips: 10.133.0.69 2026-07-29T09:27:10.004515830Z I0729 09:27:10.004453 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 2026-07-29T09:27:10.004515830Z I0729 09:27:10.004488 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-inf-b-612453/test-model-612453-kserve-686548c7d7-tmlc6 2026-07-29T09:27:10.459493561Z I0729 09:27:10.459443 3399 namespace.go:290] [ai-tenant-e2e-inf-a-306d65] deleting namespace 2026-07-29T09:27:10.969626027Z I0729 09:27:10.969545 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:27:14.090967309Z I0729 09:27:14.090927 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:27:14.091059301Z I0729 09:27:14.091024 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:27:14.091059301Z I0729 09:27:14.091043 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:27:14.091059301Z I0729 09:27:14.091052 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:27:14.091083614Z I0729 09:27:14.091059 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:27:14.091083614Z I0729 09:27:14.091069 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:27:16.993583713Z I0729 09:27:16.993532 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-inf-a-306d65-openshift-default-5cd666dc66-xg8ml 2026-07-29T09:27:16.993632707Z I0729 09:27:16.993589 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-inf-a-306d65-openshift-default-5cd666dc66-xg8ml 2026-07-29T09:27:21.670168672Z I0729 09:27:21.670131 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-inf-b-612453 2026-07-29T09:27:21.670168672Z I0729 09:27:21.670154 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-inf-b-612453 2026-07-29T09:27:21.731755582Z I0729 09:27:21.731714 3399 namespace.go:133] [ai-tenant-e2e-inf-b-612453] updating namespace 2026-07-29T09:27:21.772559143Z I0729 09:27:21.772517 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:27:21.778433294Z I0729 09:27:21.778392 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778433294Z I0729 09:27:21.778415 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778433294Z I0729 09:27:21.778424 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778465435Z I0729 09:27:21.778432 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778465435Z I0729 09:27:21.778440 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778465435Z I0729 09:27:21.778449 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778465435Z I0729 09:27:21.778460 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.778492265Z I0729 09:27:21.778477 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-inf-b-612453-5wp66 2026-07-29T09:27:21.811155917Z E0729 09:27:21.811110 3399 link_network_manager.go:104] Error while syncing link "297fc57d62fad6f": failed to get address from link "297fc57d62fad6f": failed to list addresses for "297fc57d62fad6f": no such device 2026-07-29T09:27:21.854487337Z I0729 09:27:21.854441 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 297fc57d62fad6fc30ed9a97856b944e0c4e8601ab4a875043700a0a70aabd2a network default NAD default NAD key default], result "{}", err 2026-07-29T09:27:21.899885694Z I0729 09:27:21.899848 3399 pods.go:178] Deleting pod: openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 2026-07-29T09:27:21.901487856Z I0729 09:27:21.901453 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt, ips: 10.133.0.68 2026-07-29T09:27:21.902328067Z I0729 09:27:21.902295 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:27:21.902379690Z I0729 09:27:21.902368 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:27:21.902411683Z I0729 09:27:21.902403 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:27:21.902459267Z I0729 09:27:21.902433 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:27:21.902459267Z I0729 09:27:21.902453 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:27:22.017444376Z I0729 09:27:22.017399 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.017444376Z I0729 09:27:22.017430 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.017444376Z I0729 09:27:22.017439 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.017497492Z I0729 09:27:22.017448 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.017497492Z I0729 09:27:22.017455 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.017497492Z I0729 09:27:22.017463 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.017497492Z I0729 09:27:22.017479 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-inf-b-612453-67b4fbb85d-sd5n6 2026-07-29T09:27:22.079972402Z I0729 09:27:22.079919 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 2026-07-29T09:27:22.080002555Z I0729 09:27:22.079978 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-inf-b-612453-767cf94c94-q8djt 2026-07-29T09:27:22.116818557Z I0729 09:27:22.116775 3399 namespace.go:133] [ai-tenant-e2e-inf-b-612453] updating namespace 2026-07-29T09:27:22.317989618Z I0729 09:27:22.317945 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-inf-b-612453-54c54f4f6-qvd5h 2026-07-29T09:27:22.317989618Z I0729 09:27:22.317972 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-inf-b-612453-54c54f4f6-qvd5h 2026-07-29T09:27:28.087598853Z I0729 09:27:28.087543 3399 namespace.go:133] [ai-tenant-e2e-inf-b-612453] updating namespace 2026-07-29T09:27:28.097307867Z I0729 09:27:28.097258 3399 namespace.go:290] [ai-tenant-e2e-inf-b-612453] deleting namespace 2026-07-29T09:27:28.563456778Z I0729 09:27:28.563413 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:27:28.888691137Z I0729 09:27:28.888646 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:27:30.010487499Z I0729 09:27:30.010444 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 2026-07-29T09:27:30.068873476Z W0729 09:27:30.068826 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-api-a-d0f1b8-openshift-default 2026-07-29T09:27:30.068873476Z W0729 09:27:30.068847 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-api-a-d0f1b8-openshift-default 2026-07-29T09:27:31.892704334Z I0729 09:27:31.892659 3399 namespace.go:89] [ai-tenant-e2e-api-a-d0f1b8] adding namespace 2026-07-29T09:27:31.893644467Z I0729 09:27:31.893609 3399 namespace.go:93] [ai-tenant-e2e-api-a-d0f1b8] adding namespace took 888.561µs 2026-07-29T09:27:31.910505818Z I0729 09:27:31.910459 3399 namespace.go:133] [ai-tenant-e2e-api-a-d0f1b8] updating namespace 2026-07-29T09:27:31.922009111Z I0729 09:27:31.921975 3399 namespace.go:133] [ai-tenant-e2e-api-a-d0f1b8] updating namespace 2026-07-29T09:27:32.994173875Z I0729 09:27:32.994111 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-api-a-d0f1b8 for network default 2026-07-29T09:27:32.994220816Z I0729 09:27:32.994207 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-api-a-d0f1b8 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:27:32.996530160Z I0729 09:27:32.996492 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-api-a-d0f1b8 resources completed, update namespace loglevel 2026-07-29T09:27:34.084363391Z W0729 09:27:34.084321 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8 2026-07-29T09:27:34.090608024Z I0729 09:27:34.090570 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-inf-b-612453-openshift-default-696d5c78f4-blhsq 2026-07-29T09:27:34.090685011Z I0729 09:27:34.090661 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-inf-b-612453-openshift-default-696d5c78f4-blhsq 2026-07-29T09:27:34.107813462Z W0729 09:27:34.107771 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-api-a-d0f1b8 2026-07-29T09:27:34.121494668Z W0729 09:27:34.121460 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-api-a-d0f1b8 2026-07-29T09:27:37.048609258Z I0729 09:27:37.048556 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=6 2026-07-29T09:27:40.907677401Z I0729 09:27:40.907638 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=126 2026-07-29T09:27:47.197917218Z W0729 09:27:47.197849 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-api-a-d0f1b8-openshift-default 2026-07-29T09:27:47.197917218Z W0729 09:27:47.197876 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-api-a-d0f1b8-openshift-default 2026-07-29T09:27:53.027800093Z I0729 09:27:53.027753 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=6 2026-07-29T09:27:56.878347417Z I0729 09:27:56.878298 3399 namespace.go:133] [ai-tenant-e2e-api-a-d0f1b8] updating namespace 2026-07-29T09:27:57.330000551Z I0729 09:27:57.329952 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:27:58.224480810Z W0729 09:27:58.224432 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-api-b-cddb75-openshift-default 2026-07-29T09:27:58.224572510Z W0729 09:27:58.224542 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-api-b-cddb75-openshift-default 2026-07-29T09:28:00.026754557Z I0729 09:28:00.026705 3399 namespace.go:89] [ai-tenant-e2e-api-b-cddb75] adding namespace 2026-07-29T09:28:00.027481091Z I0729 09:28:00.027450 3399 namespace.go:93] [ai-tenant-e2e-api-b-cddb75] adding namespace took 700.283µs 2026-07-29T09:28:00.042323983Z I0729 09:28:00.042287 3399 namespace.go:133] [ai-tenant-e2e-api-b-cddb75] updating namespace 2026-07-29T09:28:00.051514321Z I0729 09:28:00.051471 3399 namespace.go:133] [ai-tenant-e2e-api-b-cddb75] updating namespace 2026-07-29T09:28:01.047210258Z I0729 09:28:01.047162 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh] creating logical port openshift-ingress_payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:28:01.047328590Z I0729 09:28:01.047305 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 2026-07-29T09:28:01.062936085Z I0729 09:28:01.062875 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh] addLogicalPort annotation time took 15.601202ms 2026-07-29T09:28:01.065087929Z I0729 09:28:01.065038 3399 pods.go:273] [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh] addLogicalPort took 17.902814ms, libovsdb time 1.508668ms 2026-07-29T09:28:01.115198190Z I0729 09:28:01.115108 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-api-b-cddb75 for network default 2026-07-29T09:28:01.115234278Z I0729 09:28:01.115194 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-api-b-cddb75 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:28:01.118109740Z I0729 09:28:01.118075 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-api-b-cddb75 resources completed, update namespace loglevel 2026-07-29T09:28:01.422541396Z I0729 09:28:01.422495 3399 cniserver.go:311] [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:28:01.433031400Z I0729 09:28:01.432977 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh, hostIfaceName: 8f675abb40e9cee, network: default, NAD default, SandboxID: "8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2", PCI device ID: , UID: "e3998755-be7b-4c9a-9ae6-838673dba469", MAC: 0a:58:0a:85:00:46, IPs: [10.133.0.70/23] 2026-07-29T09:28:01.497056885Z I0729 09:28:01.497003 3399 cniserver.go:300] [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:7e:48:5c:41:ae\",\"name\":\"8f675abb40e9cee\"},{\"mac\":\"0a:58:0a:85:00:46\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/75eb7aec-950b-4c5a-9f9b-b29c83d5e62c\"}],\"ips\":[{\"address\":\"10.133.0.70/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:28:02.247692856Z W0729 09:28:02.247643 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75 2026-07-29T09:28:02.260739765Z W0729 09:28:02.260696 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-api-b-cddb75 2026-07-29T09:28:02.308720201Z W0729 09:28:02.308672 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-api-b-cddb75 2026-07-29T09:28:09.909058696Z I0729 09:28:09.909004 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=69 2026-07-29T09:28:13.317579367Z W0729 09:28:13.317519 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-api-b-cddb75 2026-07-29T09:28:15.319465013Z W0729 09:28:15.319419 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-api-b-cddb75-openshift-default 2026-07-29T09:28:15.319579108Z W0729 09:28:15.319530 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-api-b-cddb75-openshift-default 2026-07-29T09:28:25.018396893Z I0729 09:28:25.018348 3399 namespace.go:133] [ai-tenant-e2e-api-b-cddb75] updating namespace 2026-07-29T09:28:27.886491187Z I0729 09:28:27.886449 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:28:27.886659426Z I0729 09:28:27.886636 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:27.895747228Z I0729 09:28:27.895638 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd] addLogicalPort annotation time took 9.047248ms 2026-07-29T09:28:27.897503023Z I0729 09:28:27.897458 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd] addLogicalPort took 11.026467ms, libovsdb time 1.12327ms 2026-07-29T09:28:28.254510560Z I0729 09:28:28.254463 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:28:28.265211073Z I0729 09:28:28.265164 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd, hostIfaceName: 4d1af2517133f25, network: default, NAD default, SandboxID: "4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be", PCI device ID: , UID: "e0044770-17bb-4c34-b5e6-4ccc667e7d9e", MAC: 0a:58:0a:85:00:47, IPs: [10.133.0.71/23] 2026-07-29T09:28:28.324697792Z I0729 09:28:28.324641 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:be:6f:bb:0e:73\",\"name\":\"4d1af2517133f25\"},{\"mac\":\"0a:58:0a:85:00:47\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b5ff29f-79e2-43e1-b639-cc433bcb9943\"}],\"ips\":[{\"address\":\"10.133.0.71/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:28:30.501802651Z I0729 09:28:30.501755 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:28:30.527747449Z E0729 09:28:30.527696 3399 link_network_manager.go:104] Error while syncing link "4d1af2517133f25": failed to get address from link "4d1af2517133f25": failed to list addresses for "4d1af2517133f25": no such device 2026-07-29T09:28:30.586939735Z I0729 09:28:30.586865 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 4d1af2517133f254452aaea38cf5b25845c78ecbae73e66a439399d0f68921be network default NAD default NAD key default], result "{}", err 2026-07-29T09:28:30.630533536Z I0729 09:28:30.630466 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:30.632533729Z I0729 09:28:30.632493 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd, ips: 10.133.0.71 2026-07-29T09:28:30.632559689Z I0729 09:28:30.632536 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:28:30.632759299Z I0729 09:28:30.632738 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:28:30.632995779Z I0729 09:28:30.632974 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:28:30.633194830Z I0729 09:28:30.633178 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:28:30.633420164Z I0729 09:28:30.633397 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:28:30.633571359Z I0729 09:28:30.633558 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:28:43.479853925Z I0729 09:28:43.479785 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-api-a-d0f1b8 2026-07-29T09:28:43.479952065Z I0729 09:28:43.479932 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-api-a-d0f1b8 2026-07-29T09:28:43.535803761Z I0729 09:28:43.535759 3399 namespace.go:133] [ai-tenant-e2e-api-a-d0f1b8] updating namespace 2026-07-29T09:28:43.585580442Z I0729 09:28:43.585532 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585636961Z I0729 09:28:43.585624 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585673838Z I0729 09:28:43.585656 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585684281Z I0729 09:28:43.585672 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585684281Z I0729 09:28:43.585679 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585695897Z I0729 09:28:43.585689 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585707386Z I0729 09:28:43.585700 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.585737293Z I0729 09:28:43.585720 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-a-d0f1b8-7sgwd 2026-07-29T09:28:43.731261201Z I0729 09:28:43.731206 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:28:43.731261201Z I0729 09:28:43.731242 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:28:43.731261201Z I0729 09:28:43.731255 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:28:43.731300806Z I0729 09:28:43.731267 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:28:43.731300806Z I0729 09:28:43.731283 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:28:43.907724196Z I0729 09:28:43.907682 3399 namespace.go:133] [ai-tenant-e2e-api-a-d0f1b8] updating namespace 2026-07-29T09:28:44.481755903Z I0729 09:28:44.481711 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-api-a-d0f1b8-77c457db94-p2kft 2026-07-29T09:28:44.481755903Z I0729 09:28:44.481741 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-api-a-d0f1b8-77c457db94-p2kft 2026-07-29T09:28:44.494835295Z I0729 09:28:44.494798 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.494835295Z I0729 09:28:44.494821 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.494835295Z I0729 09:28:44.494829 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.494871930Z I0729 09:28:44.494835 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.494871930Z I0729 09:28:44.494843 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.494871930Z I0729 09:28:44.494851 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.494871930Z I0729 09:28:44.494866 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-api-a-d0f1b8-5b49bc56bd-cm5z2 2026-07-29T09:28:44.514022381Z I0729 09:28:44.513985 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-api-a-d0f1b8-7ffd78c7bd-ts8cg 2026-07-29T09:28:44.514022381Z I0729 09:28:44.514010 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-api-a-d0f1b8-7ffd78c7bd-ts8cg 2026-07-29T09:28:49.789140403Z I0729 09:28:49.789081 3399 namespace.go:133] [ai-tenant-e2e-api-a-d0f1b8] updating namespace 2026-07-29T09:28:49.802109460Z I0729 09:28:49.802076 3399 namespace.go:290] [ai-tenant-e2e-api-a-d0f1b8] deleting namespace 2026-07-29T09:28:50.265411555Z I0729 09:28:50.265368 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:28:50.426316791Z I0729 09:28:50.426268 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:28:50.426439192Z I0729 09:28:50.426417 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:28:50.436102632Z I0729 09:28:50.436065 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7] addLogicalPort annotation time took 9.680247ms 2026-07-29T09:28:50.437841582Z I0729 09:28:50.437666 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7] addLogicalPort took 11.411648ms, libovsdb time 982.473µs 2026-07-29T09:28:50.794742020Z I0729 09:28:50.794695 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:28:50.804766410Z I0729 09:28:50.804728 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-api-b-cddb75-blqv7, hostIfaceName: 14ae56597f7574a, network: default, NAD default, SandboxID: "14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d", PCI device ID: , UID: "45372243-9c77-4b9c-a887-611376a05edb", MAC: 0a:58:0a:85:00:48, IPs: [10.133.0.72/23] 2026-07-29T09:28:51.074216253Z I0729 09:28:51.074162 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:e4:c6:1f:5c:44\",\"name\":\"14ae56597f7574a\"},{\"mac\":\"0a:58:0a:85:00:48\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a70cb5ed-7306-42eb-9120-b270f5edd51e\"}],\"ips\":[{\"address\":\"10.133.0.72/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:28:53.646190041Z I0729 09:28:53.646145 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:28:53.670408902Z E0729 09:28:53.670371 3399 link_network_manager.go:104] Error while syncing link "14ae56597f7574a": failed to get address from link "14ae56597f7574a": failed to list addresses for "14ae56597f7574a": no such device 2026-07-29T09:28:53.734255058Z I0729 09:28:53.734197 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 14ae56597f7574a2003e6f612a9eb425d4f2b68b07812ca64295c847437e7a3d network default NAD default NAD key default], result "{}", err 2026-07-29T09:28:53.776711436Z I0729 09:28:53.776666 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:28:53.778468217Z I0729 09:28:53.778429 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7, ips: 10.133.0.72 2026-07-29T09:28:53.778492609Z I0729 09:28:53.778481 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:28:53.778757680Z I0729 09:28:53.778742 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:28:53.778946203Z I0729 09:28:53.778927 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:28:53.779164318Z I0729 09:28:53.779145 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:28:53.779325866Z I0729 09:28:53.779310 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:28:53.779505785Z I0729 09:28:53.779481 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:28:55.532448771Z I0729 09:28:55.532400 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-api-a-d0f1b8-openshift-default-6c7d789b6-4h527 2026-07-29T09:28:55.532498720Z I0729 09:28:55.532486 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-api-a-d0f1b8-openshift-default-6c7d789b6-4h527 2026-07-29T09:29:06.034392958Z I0729 09:29:06.034313 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-api-b-cddb75 2026-07-29T09:29:06.034392958Z I0729 09:29:06.034360 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-api-b-cddb75 2026-07-29T09:29:06.094750073Z I0729 09:29:06.094705 3399 namespace.go:133] [ai-tenant-e2e-api-b-cddb75] updating namespace 2026-07-29T09:29:06.139471276Z I0729 09:29:06.139421 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139546328Z I0729 09:29:06.139534 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139586625Z I0729 09:29:06.139576 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139614513Z I0729 09:29:06.139605 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139640063Z I0729 09:29:06.139631 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139664018Z I0729 09:29:06.139655 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139690411Z I0729 09:29:06.139681 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.139722077Z I0729 09:29:06.139713 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-api-b-cddb75-blqv7 2026-07-29T09:29:06.146791849Z I0729 09:29:06.146749 3399 cniserver.go:311] [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:29:06.187383023Z E0729 09:29:06.187335 3399 link_network_manager.go:104] Error while syncing link "8f675abb40e9cee": failed to get address from link "8f675abb40e9cee": failed to list addresses for "8f675abb40e9cee": no such device 2026-07-29T09:29:06.233364509Z I0729 09:29:06.233314 3399 cniserver.go:300] [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 8f675abb40e9cee94bcefe1641ef63708f3ed6a31fcea295e790fdfc025cd2f2 network default NAD default NAD key default], result "{}", err 2026-07-29T09:29:06.249930023Z I0729 09:29:06.249878 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:29:06.249930023Z I0729 09:29:06.249917 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:29:06.249957739Z I0729 09:29:06.249931 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:29:06.249957739Z I0729 09:29:06.249942 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:29:06.249966721Z I0729 09:29:06.249956 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:29:06.277035200Z I0729 09:29:06.276992 3399 pods.go:178] Deleting pod: openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 2026-07-29T09:29:06.278621566Z I0729 09:29:06.278582 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh, ips: 10.133.0.70 2026-07-29T09:29:06.473176865Z I0729 09:29:06.473131 3399 namespace.go:133] [ai-tenant-e2e-api-b-cddb75] updating namespace 2026-07-29T09:29:06.583908190Z I0729 09:29:06.583843 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.583908190Z I0729 09:29:06.583868 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.583908190Z I0729 09:29:06.583874 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.583908190Z I0729 09:29:06.583879 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.583908190Z I0729 09:29:06.583885 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.583949218Z I0729 09:29:06.583915 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.583949218Z I0729 09:29:06.583932 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-api-b-cddb75-57cf7fd6d5-ck2z8 2026-07-29T09:29:06.595065447Z I0729 09:29:06.594966 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-api-b-cddb75-79f4c86d44-nz6hd 2026-07-29T09:29:06.595065447Z I0729 09:29:06.594998 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-api-b-cddb75-79f4c86d44-nz6hd 2026-07-29T09:29:06.700113225Z I0729 09:29:06.700071 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 2026-07-29T09:29:06.700144599Z I0729 09:29:06.700122 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-api-b-cddb75-79cb485ff6-98chh 2026-07-29T09:29:12.181014626Z I0729 09:29:12.180953 3399 namespace.go:133] [ai-tenant-e2e-api-b-cddb75] updating namespace 2026-07-29T09:29:12.190018563Z I0729 09:29:12.189975 3399 namespace.go:290] [ai-tenant-e2e-api-b-cddb75] deleting namespace 2026-07-29T09:29:12.669922938Z I0729 09:29:12.669863 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:29:13.008672925Z I0729 09:29:13.008625 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:29:14.629786714Z W0729 09:29:14.629745 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-shared-a-0abb48-openshift-default 2026-07-29T09:29:14.629835943Z W0729 09:29:14.629807 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-shared-a-0abb48-openshift-default 2026-07-29T09:29:16.462739291Z I0729 09:29:16.462692 3399 namespace.go:89] [ai-tenant-e2e-shared-a-0abb48] adding namespace 2026-07-29T09:29:16.463528401Z I0729 09:29:16.463493 3399 namespace.go:93] [ai-tenant-e2e-shared-a-0abb48] adding namespace took 724.678µs 2026-07-29T09:29:16.481985460Z I0729 09:29:16.481945 3399 namespace.go:133] [ai-tenant-e2e-shared-a-0abb48] updating namespace 2026-07-29T09:29:16.492084090Z I0729 09:29:16.492042 3399 namespace.go:133] [ai-tenant-e2e-shared-a-0abb48] updating namespace 2026-07-29T09:29:17.646608928Z W0729 09:29:17.646560 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48 2026-07-29T09:29:17.715372867Z I0729 09:29:17.715331 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-shared-a-0abb48 for network default 2026-07-29T09:29:17.715524131Z I0729 09:29:17.715478 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-shared-a-0abb48 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:29:17.717435632Z I0729 09:29:17.717411 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-shared-a-0abb48 resources completed, update namespace loglevel 2026-07-29T09:29:18.653916093Z W0729 09:29:18.653849 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-shared-a-0abb48 2026-07-29T09:29:18.662724945Z I0729 09:29:18.662683 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-api-b-cddb75-openshift-default-6f4c8c56dd-x6f7f 2026-07-29T09:29:18.662786329Z I0729 09:29:18.662774 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-api-b-cddb75-openshift-default-6f4c8c56dd-x6f7f 2026-07-29T09:29:18.684067815Z W0729 09:29:18.684016 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-shared-a-0abb48 2026-07-29T09:29:30.724170976Z W0729 09:29:30.724121 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-shared-a-0abb48-openshift-default 2026-07-29T09:29:30.724170976Z W0729 09:29:30.724150 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-shared-a-0abb48-openshift-default 2026-07-29T09:29:41.438734514Z I0729 09:29:41.438687 3399 namespace.go:133] [ai-tenant-e2e-shared-a-0abb48] updating namespace 2026-07-29T09:29:41.770337282Z I0729 09:29:41.770289 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:29:42.791755713Z W0729 09:29:42.791706 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-shared-b-27430b-openshift-default 2026-07-29T09:29:42.791755713Z W0729 09:29:42.791730 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-shared-b-27430b-openshift-default 2026-07-29T09:29:44.601055870Z I0729 09:29:44.601006 3399 namespace.go:89] [ai-tenant-e2e-shared-b-27430b] adding namespace 2026-07-29T09:29:44.601733099Z I0729 09:29:44.601700 3399 namespace.go:93] [ai-tenant-e2e-shared-b-27430b] adding namespace took 637.932µs 2026-07-29T09:29:44.610852660Z I0729 09:29:44.610818 3399 namespace.go:133] [ai-tenant-e2e-shared-b-27430b] updating namespace 2026-07-29T09:29:44.622798990Z I0729 09:29:44.622746 3399 namespace.go:133] [ai-tenant-e2e-shared-b-27430b] updating namespace 2026-07-29T09:29:45.615840316Z I0729 09:29:45.615780 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b] creating logical port openshift-ingress_payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:29:45.615976833Z I0729 09:29:45.615944 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b 2026-07-29T09:29:45.656348800Z I0729 09:29:45.656301 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b] addLogicalPort annotation time took 40.415269ms 2026-07-29T09:29:45.658796563Z I0729 09:29:45.658754 3399 pods.go:273] [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b] addLogicalPort took 42.96163ms, libovsdb time 1.823812ms 2026-07-29T09:29:45.743986323Z I0729 09:29:45.743941 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-shared-b-27430b for network default 2026-07-29T09:29:45.744112182Z I0729 09:29:45.744088 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-shared-b-27430b added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:29:45.746380726Z I0729 09:29:45.746347 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-shared-b-27430b resources completed, update namespace loglevel 2026-07-29T09:29:45.983055610Z I0729 09:29:45.983008 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:29:45.993640871Z I0729 09:29:45.993591 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b, hostIfaceName: dc25acd3edc06df, network: default, NAD default, SandboxID: "dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70", PCI device ID: , UID: "3468b53b-2eff-410b-926f-47458c021a67", MAC: 0a:58:0a:85:00:49, IPs: [10.133.0.73/23] 2026-07-29T09:29:46.260364253Z I0729 09:29:46.260303 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:ae:22:b4:4d:90\",\"name\":\"dc25acd3edc06df\"},{\"mac\":\"0a:58:0a:85:00:49\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ab6b14cf-211e-499d-9aab-43850bdf4d91\"}],\"ips\":[{\"address\":\"10.133.0.73/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:29:46.821694700Z W0729 09:29:46.821644 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b 2026-07-29T09:29:46.860208724Z W0729 09:29:46.860149 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-shared-b-27430b 2026-07-29T09:29:46.962956297Z W0729 09:29:46.962916 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-shared-b-27430b 2026-07-29T09:29:56.034659963Z I0729 09:29:56.034607 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=9 2026-07-29T09:29:57.948609821Z W0729 09:29:57.948566 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-shared-b-27430b 2026-07-29T09:29:59.885251325Z W0729 09:29:59.885210 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-shared-b-27430b-openshift-default 2026-07-29T09:29:59.885251325Z W0729 09:29:59.885230 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-shared-b-27430b-openshift-default 2026-07-29T09:30:00.130178687Z I0729 09:30:00.130136 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf] creating logical port odh-ai-gateway-infra_maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:30:00.130290060Z I0729 09:30:00.130268 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:30:00.130429363Z I0729 09:30:00.130303 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6] creating logical port odh-ai-gateway-infra_maas-api-key-cleanup-29755290-g2lk6 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:30:00.130595121Z I0729 09:30:00.130572 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 2026-07-29T09:30:00.139837330Z I0729 09:30:00.139799 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6] addLogicalPort annotation time took 9.25218ms 2026-07-29T09:30:00.141881462Z I0729 09:30:00.141849 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6] addLogicalPort took 11.551569ms, libovsdb time 1.221108ms 2026-07-29T09:30:00.143038054Z I0729 09:30:00.143010 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf] addLogicalPort annotation time took 12.790537ms 2026-07-29T09:30:00.144952023Z I0729 09:30:00.144926 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf] addLogicalPort took 14.806979ms, libovsdb time 1.15695ms 2026-07-29T09:30:00.209848847Z I0729 09:30:00.209803 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:30:00.216747901Z I0729 09:30:00.216699 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:30:00.223223332Z I0729 09:30:00.223175 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf, hostIfaceName: 8b0d2ce5f233860, network: default, NAD default, SandboxID: "8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73", PCI device ID: , UID: "8331856f-7fe4-4792-bed5-c1e6301ce751", MAC: 0a:58:0a:85:00:4a, IPs: [10.133.0.74/23] 2026-07-29T09:30:00.230630544Z I0729 09:30:00.230587 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-key-cleanup-29755290-g2lk6, hostIfaceName: 045f310cc68b932, network: default, NAD default, SandboxID: "045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b", PCI device ID: , UID: "760b17ee-e769-4eb2-8c79-2cf29837d210", MAC: 0a:58:0a:85:00:4b, IPs: [10.133.0.75/23] 2026-07-29T09:30:00.311130131Z I0729 09:30:00.311066 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:d8:29:45:73:62\",\"name\":\"045f310cc68b932\"},{\"mac\":\"0a:58:0a:85:00:4b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d47f62a-0de5-4120-84be-5a834282c20b\"}],\"ips\":[{\"address\":\"10.133.0.75/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:30:00.495927384Z I0729 09:30:00.495860 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73 network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:61:5a:b7:cc:fe\",\"name\":\"8b0d2ce5f233860\"},{\"mac\":\"0a:58:0a:85:00:4a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e608df95-cedf-4c3b-9eee-63759d3e0a57\"}],\"ips\":[{\"address\":\"10.133.0.74/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:30:03.061498711Z I0729 09:30:03.061447 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:30:03.061498711Z I0729 09:30:03.061477 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:30:03.061852905Z I0729 09:30:03.061813 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:30:03.061852905Z I0729 09:30:03.061843 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:30:03.061906459Z I0729 09:30:03.061860 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:30:03.061906459Z I0729 09:30:03.061871 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:30:03.061906459Z I0729 09:30:03.061881 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:30:03.061925788Z I0729 09:30:03.061919 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:30:03.102421719Z E0729 09:30:03.102381 3399 link_network_manager.go:104] Error while syncing link "045f310cc68b932": failed to get address from link "045f310cc68b932": failed to list addresses for "045f310cc68b932": no such device 2026-07-29T09:30:03.127920607Z E0729 09:30:03.127863 3399 link_network_manager.go:104] Error while syncing link "8b0d2ce5f233860": failed to get address from link "8b0d2ce5f233860": failed to list addresses for "8b0d2ce5f233860": no such device 2026-07-29T09:30:03.145284635Z I0729 09:30:03.145233 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 045f310cc68b932fe5576ecdd2292a11e92ca6293c6f7a5e6ddb872acfaa203b network default NAD default NAD key default], result "{}", err 2026-07-29T09:30:03.177080148Z I0729 09:30:03.177028 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73 network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 8b0d2ce5f233860f94c5fc18e46e89ec01bbc3559bb2ae90d49e757c15c5cc73 network default NAD default NAD key default], result "{}", err 2026-07-29T09:30:03.187088755Z I0729 09:30:03.187053 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6 2026-07-29T09:30:03.189396639Z I0729 09:30:03.189363 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-key-cleanup-29755290-g2lk6, ips: 10.133.0.75 2026-07-29T09:30:03.189491557Z I0729 09:30:03.189462 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:30:03.189770751Z I0729 09:30:03.189748 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:30:03.190078798Z I0729 09:30:03.190042 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:30:03.190301951Z I0729 09:30:03.190275 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:30:03.190516426Z I0729 09:30:03.190496 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:30:03.190909767Z I0729 09:30:03.190869 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:30:03.227028317Z I0729 09:30:03.226986 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:30:03.229124547Z I0729 09:30:03.229092 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf, ips: 10.133.0.74 2026-07-29T09:30:03.229152799Z I0729 09:30:03.229144 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:30:03.229426921Z I0729 09:30:03.229405 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:30:03.229646975Z I0729 09:30:03.229626 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:30:03.229843637Z I0729 09:30:03.229825 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:30:03.230080979Z I0729 09:30:03.230059 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:30:03.230345351Z I0729 09:30:03.230314 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:30:09.585272353Z I0729 09:30:09.585220 3399 namespace.go:133] [ai-tenant-e2e-shared-b-27430b] updating namespace 2026-07-29T09:30:31.026634580Z I0729 09:30:31.026589 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=7 2026-07-29T09:30:32.915805750Z I0729 09:30:32.915747 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2] creating logical port ai-tenant-e2e-shared-a-0abb48_auth-test-model-0abb48-kserve-6544d444ff-xczn2 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:30:32.915981064Z I0729 09:30:32.915948 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 2026-07-29T09:30:32.926023677Z I0729 09:30:32.925981 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2] addLogicalPort annotation time took 10.106635ms 2026-07-29T09:30:32.927691807Z I0729 09:30:32.927657 3399 pods.go:273] [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2] addLogicalPort took 11.928933ms, libovsdb time 1.09097ms 2026-07-29T09:30:33.283200164Z I0729 09:30:33.283154 3399 cniserver.go:311] [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:30:33.293498062Z I0729 09:30:33.293443 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-shared-a-0abb48, podName: auth-test-model-0abb48-kserve-6544d444ff-xczn2, hostIfaceName: c84197a00e6d6ae, network: default, NAD default, SandboxID: "c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330", PCI device ID: , UID: "c54f0233-0c57-4f9e-a947-aa8eca3cf0de", MAC: 0a:58:0a:85:00:4c, IPs: [10.133.0.76/23] 2026-07-29T09:30:33.360321056Z I0729 09:30:33.360276 3399 cniserver.go:300] [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330 network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:38:c1:fa:e1:4f\",\"name\":\"c84197a00e6d6ae\"},{\"mac\":\"0a:58:0a:85:00:4c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/144c96b9-82a5-4372-a5c4-8aa59ffdf740\"}],\"ips\":[{\"address\":\"10.133.0.76/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:30:34.238204409Z W0729 09:30:34.238154 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-workload-svc 2026-07-29T09:30:37.009391985Z I0729 09:30:37.009342 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 2026-07-29T09:30:43.296925414Z W0729 09:30:43.296850 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-workload-svc 2026-07-29T09:30:54.047045853Z I0729 09:30:54.047000 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 2026-07-29T09:31:02.521447133Z I0729 09:31:02.521404 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9] creating logical port ai-tenant-e2e-shared-b-27430b_auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:31:02.521668640Z I0729 09:31:02.521643 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 2026-07-29T09:31:02.533424498Z I0729 09:31:02.533378 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9] addLogicalPort annotation time took 11.782422ms 2026-07-29T09:31:02.535019989Z I0729 09:31:02.534974 3399 pods.go:273] [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9] addLogicalPort took 13.588042ms, libovsdb time 897.44µs 2026-07-29T09:31:02.894435326Z I0729 09:31:02.894389 3399 cniserver.go:311] [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:31:02.904052579Z I0729 09:31:02.904002 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-shared-b-27430b, podName: auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9, hostIfaceName: 4d70c0a14111404, network: default, NAD default, SandboxID: "4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383", PCI device ID: , UID: "33849953-278a-4e0e-87af-758e6225d2ae", MAC: 0a:58:0a:85:00:4d, IPs: [10.133.0.77/23] 2026-07-29T09:31:02.965456371Z I0729 09:31:02.965407 3399 cniserver.go:300] [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383 network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:6a:7e:73:c9:98\",\"name\":\"4d70c0a14111404\"},{\"mac\":\"0a:58:0a:85:00:4d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c606a5c0-e8c4-414d-abd5-d2a752b46997\"}],\"ips\":[{\"address\":\"10.133.0.77/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:31:03.409536781Z W0729 09:31:03.409481 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-workload-svc 2026-07-29T09:31:07.040444728Z I0729 09:31:07.040390 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 2026-07-29T09:31:11.052713872Z I0729 09:31:11.052666 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=6 2026-07-29T09:31:13.480749307Z W0729 09:31:13.480700 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-workload-svc 2026-07-29T09:31:39.909673867Z I0729 09:31:39.909621 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=415 2026-07-29T09:31:46.017054955Z I0729 09:31:46.017000 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressIP" totalItems=10 2026-07-29T09:32:56.051158525Z I0729 09:32:56.051113 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=7 2026-07-29T09:33:35.911331645Z I0729 09:33:35.911284 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=28 2026-07-29T09:33:52.910172694Z I0729 09:33:52.910127 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=117 2026-07-29T09:34:57.910132528Z I0729 09:34:57.910081 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=42 2026-07-29T09:35:01.911023696Z I0729 09:35:01.910985 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=49 2026-07-29T09:35:35.781167850Z W0729 09:35:35.781118 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-shared-a-0abb48/sub-test-model-0abb48-kserve-workload-svc 2026-07-29T09:35:39.998439557Z I0729 09:35:39.998389 3399 cniserver.go:311] [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:35:40.022311980Z E0729 09:35:40.022267 3399 link_network_manager.go:104] Error while syncing link "c84197a00e6d6ae": failed to get address from link "c84197a00e6d6ae": failed to list addresses for "c84197a00e6d6ae": no such device 2026-07-29T09:35:40.078641644Z I0729 09:35:40.078589 3399 cniserver.go:300] [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330 network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 c84197a00e6d6ae926bae8701ab7c2932a026e55e44aa3c5e6ffc098dbce8330 network default NAD default NAD key default], result "{}", err 2026-07-29T09:35:40.127792881Z I0729 09:35:40.127740 3399 pods.go:178] Deleting pod: ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 2026-07-29T09:35:40.129366888Z I0729 09:35:40.129323 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2, ips: 10.133.0.76 2026-07-29T09:35:40.894805848Z I0729 09:35:40.894750 3399 cniserver.go:311] [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:35:40.916314983Z E0729 09:35:40.916276 3399 link_network_manager.go:104] Error while syncing link "4d70c0a14111404": failed to get address from link "4d70c0a14111404": failed to list addresses for "4d70c0a14111404": no such device 2026-07-29T09:35:40.973120314Z I0729 09:35:40.973064 3399 cniserver.go:300] [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383 network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 4d70c0a14111404e79d2ad0975d99e16744de8281d45bb95a6697f6e5595f383 network default NAD default NAD key default], result "{}", err 2026-07-29T09:35:41.018603853Z I0729 09:35:41.018554 3399 pods.go:178] Deleting pod: ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 2026-07-29T09:35:41.020587647Z I0729 09:35:41.020549 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9, ips: 10.133.0.77 2026-07-29T09:35:41.073601787Z I0729 09:35:41.073550 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 2026-07-29T09:35:41.073601787Z I0729 09:35:41.073588 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-shared-a-0abb48/auth-test-model-0abb48-kserve-6544d444ff-xczn2 2026-07-29T09:35:41.382165014Z I0729 09:35:41.382112 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 2026-07-29T09:35:41.382261093Z I0729 09:35:41.382228 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-shared-b-27430b/auth-test-model-27430b-kserve-7c9fcc47bc-bs7z9 2026-07-29T09:35:55.029808524Z I0729 09:35:55.029756 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=9 2026-07-29T09:36:09.506949391Z I0729 09:36:09.506883 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9] creating logical port ai-tenant-e2e-shared-b-27430b_sub-test-model-27430b-kserve-75969d5ccf-5prt9 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:36:09.507210395Z I0729 09:36:09.507180 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 2026-07-29T09:36:09.517762946Z I0729 09:36:09.517720 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9] addLogicalPort annotation time took 10.599777ms 2026-07-29T09:36:09.519171003Z I0729 09:36:09.519131 3399 pods.go:273] [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9] addLogicalPort took 12.265925ms, libovsdb time 870.225µs 2026-07-29T09:36:09.879719249Z I0729 09:36:09.879674 3399 cniserver.go:311] [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:36:09.890250801Z I0729 09:36:09.890199 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-shared-b-27430b, podName: sub-test-model-27430b-kserve-75969d5ccf-5prt9, hostIfaceName: 8ae9b6f68cdb1a8, network: default, NAD default, SandboxID: "8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37", PCI device ID: , UID: "e3049114-ed49-4369-abb2-ba925db7b104", MAC: 0a:58:0a:85:00:4e, IPs: [10.133.0.78/23] 2026-07-29T09:36:09.949707132Z I0729 09:36:09.949654 3399 cniserver.go:300] [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37 network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:2b:f0:fc:11:80\",\"name\":\"8ae9b6f68cdb1a8\"},{\"mac\":\"0a:58:0a:85:00:4e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f5d0fb8-fa03-48c1-acc9-22504674e899\"}],\"ips\":[{\"address\":\"10.133.0.78/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:36:10.249373762Z W0729 09:36:10.249325 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-workload-svc 2026-07-29T09:36:20.333543241Z W0729 09:36:20.333449 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-workload-svc 2026-07-29T09:36:24.054409278Z I0729 09:36:24.054356 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=6 2026-07-29T09:37:03.012236582Z I0729 09:37:03.012193 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 2026-07-29T09:37:05.042158823Z I0729 09:37:05.042112 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:129" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 2026-07-29T09:37:12.011406359Z I0729 09:37:12.011356 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 2026-07-29T09:37:20.909268448Z I0729 09:37:20.909232 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=116 2026-07-29T09:37:40.739727757Z I0729 09:37:40.739676 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2] creating logical port ai-tenant-e2e-shared-a-0abb48_rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:37:40.739839217Z I0729 09:37:40.739811 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 2026-07-29T09:37:40.762643351Z I0729 09:37:40.762598 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2] addLogicalPort annotation time took 22.820996ms 2026-07-29T09:37:40.765314124Z I0729 09:37:40.765273 3399 pods.go:273] [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2] addLogicalPort took 25.610313ms, libovsdb time 1.993423ms 2026-07-29T09:37:41.106053058Z I0729 09:37:41.106001 3399 cniserver.go:311] [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:37:41.116326081Z I0729 09:37:41.116273 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-shared-a-0abb48, podName: rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2, hostIfaceName: 5d6e2e83f9ed7a0, network: default, NAD default, SandboxID: "5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0", PCI device ID: , UID: "726fa7ec-6f72-4e12-8ae3-568d1acc768d", MAC: 0a:58:0a:85:00:4f, IPs: [10.133.0.79/23] 2026-07-29T09:37:41.175874838Z I0729 09:37:41.175824 3399 cniserver.go:300] [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0 network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:14:f2:c8:ee:46\",\"name\":\"5d6e2e83f9ed7a0\"},{\"mac\":\"0a:58:0a:85:00:4f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5eca51c3-9b63-4a40-97d0-83cbb04ddf24\"}],\"ips\":[{\"address\":\"10.133.0.79/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:37:41.805492113Z W0729 09:37:41.805413 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-workload-svc 2026-07-29T09:37:46.388996392Z I0729 09:37:46.388947 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-shared-a-0abb48/sub-test-model-0abb48-kserve-6696fddd9-sdsrv 2026-07-29T09:37:46.389080744Z I0729 09:37:46.389056 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-shared-a-0abb48/sub-test-model-0abb48-kserve-6696fddd9-sdsrv 2026-07-29T09:37:46.496645957Z I0729 09:37:46.496600 3399 cniserver.go:311] [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:37:46.519675319Z E0729 09:37:46.519637 3399 link_network_manager.go:104] Error while syncing link "8ae9b6f68cdb1a8": failed to get address from link "8ae9b6f68cdb1a8": failed to list addresses for "8ae9b6f68cdb1a8": no such device 2026-07-29T09:37:46.575968773Z I0729 09:37:46.575918 3399 cniserver.go:300] [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37 network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 8ae9b6f68cdb1a8b55ebfa692025a488f6e2589632874cb9e6e3a5000da9ed37 network default NAD default NAD key default], result "{}", err 2026-07-29T09:37:46.619642463Z I0729 09:37:46.619595 3399 pods.go:178] Deleting pod: ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 2026-07-29T09:37:46.621092877Z I0729 09:37:46.621061 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9, ips: 10.133.0.78 2026-07-29T09:37:46.836765591Z I0729 09:37:46.836717 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 2026-07-29T09:37:46.836884186Z I0729 09:37:46.836856 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-shared-b-27430b/sub-test-model-27430b-kserve-75969d5ccf-5prt9 2026-07-29T09:37:51.881103440Z W0729 09:37:51.881051 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-workload-svc 2026-07-29T09:37:57.522284689Z I0729 09:37:57.522243 3399 base_network_controller_pods.go:490] [default/ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb] creating logical port ai-tenant-e2e-shared-b-27430b_rate-test-model-27430b-kserve-559c4784bc-qpwrb for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:37:57.522457476Z I0729 09:37:57.522431 3399 kube.go:234] Patching annotations on pod ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 2026-07-29T09:37:57.533473213Z I0729 09:37:57.533421 3399 base_network_controller_pods.go:976] [default/ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb] addLogicalPort annotation time took 11.023469ms 2026-07-29T09:37:57.535057835Z I0729 09:37:57.534994 3399 pods.go:273] [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb] addLogicalPort took 12.767944ms, libovsdb time 986.462µs 2026-07-29T09:37:57.892421669Z I0729 09:37:57.892375 3399 cniserver.go:311] [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:37:57.902714877Z I0729 09:37:57.902659 3399 helper_linux.go:533] ConfigureOVS: namespace: ai-tenant-e2e-shared-b-27430b, podName: rate-test-model-27430b-kserve-559c4784bc-qpwrb, hostIfaceName: 1ea79daac3163e2, network: default, NAD default, SandboxID: "1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb", PCI device ID: , UID: "ba6f6608-6d39-480b-9bf1-111bbc40acc0", MAC: 0a:58:0a:85:00:50, IPs: [10.133.0.80/23] 2026-07-29T09:37:57.963593015Z I0729 09:37:57.963507 3399 cniserver.go:300] [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb network default NAD default NAD key default] ADD finished CNI request [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:b5:8c:98:4f:9e\",\"name\":\"1ea79daac3163e2\"},{\"mac\":\"0a:58:0a:85:00:50\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8facd7c2-bc87-48b9-a566-45f4c1c39448\"}],\"ips\":[{\"address\":\"10.133.0.80/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:37:58.905395228Z W0729 09:37:58.905336 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-workload-svc 2026-07-29T09:38:07.986273612Z W0729 09:38:07.986225 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8000: empty IP address endpoints for service ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-workload-svc 2026-07-29T09:38:12.049612901Z I0729 09:38:12.049557 3399 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 2026-07-29T09:38:49.036456926Z I0729 09:38:49.036408 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=9 2026-07-29T09:39:11.913219021Z I0729 09:39:11.913157 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:39:11.956791905Z I0729 09:39:11.956739 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/test-curl-7783-139681293146672 2026-07-29T09:39:11.956840726Z I0729 09:39:11.956804 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/test-curl-7783-139681293146672 2026-07-29T09:39:11.956840726Z I0729 09:39:11.956827 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/test-curl-7783-139681293146672 2026-07-29T09:39:14.931059744Z I0729 09:39:14.931015 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:39:14.958019318Z I0729 09:39:14.957956 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/test-curl-7783-139681293136432 2026-07-29T09:39:14.958019318Z I0729 09:39:14.957986 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/test-curl-7783-139681293136432 2026-07-29T09:39:14.958019318Z I0729 09:39:14.958001 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/test-curl-7783-139681293136432 2026-07-29T09:39:17.955471191Z I0729 09:39:17.955420 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:39:17.981858845Z I0729 09:39:17.981810 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/test-curl-7783-139681293117456 2026-07-29T09:39:17.981858845Z I0729 09:39:17.981845 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/test-curl-7783-139681293117456 2026-07-29T09:39:17.981889149Z I0729 09:39:17.981860 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/test-curl-7783-139681293117456 2026-07-29T09:39:20.972685304Z I0729 09:39:20.972624 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:39:21.001082401Z I0729 09:39:21.001040 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/test-curl-7783-139681293117456 2026-07-29T09:39:21.001082401Z I0729 09:39:21.001068 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/test-curl-7783-139681293117456 2026-07-29T09:39:21.001116945Z I0729 09:39:21.001080 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/test-curl-7783-139681293117456 2026-07-29T09:39:21.772325323Z I0729 09:39:21.772276 3399 cniserver.go:311] [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:39:21.795668401Z E0729 09:39:21.795626 3399 link_network_manager.go:104] Error while syncing link "5d6e2e83f9ed7a0": failed to get address from link "5d6e2e83f9ed7a0": failed to list addresses for "5d6e2e83f9ed7a0": no such device 2026-07-29T09:39:21.845389993Z I0729 09:39:21.845341 3399 cniserver.go:300] [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0 network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 5d6e2e83f9ed7a0de1ad6795830f66bd101e415cebec9865fb630752e63507a0 network default NAD default NAD key default], result "{}", err 2026-07-29T09:39:21.889124068Z I0729 09:39:21.889080 3399 pods.go:178] Deleting pod: ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 2026-07-29T09:39:21.890934795Z I0729 09:39:21.890873 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2, ips: 10.133.0.79 2026-07-29T09:39:22.404977860Z I0729 09:39:22.404931 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 2026-07-29T09:39:22.405007913Z I0729 09:39:22.404975 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-shared-a-0abb48/rate-test-model-0abb48-kserve-d4c5b8c5-hs4l2 2026-07-29T09:39:22.785354158Z I0729 09:39:22.785298 3399 cniserver.go:311] [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:39:22.809721000Z E0729 09:39:22.809669 3399 link_network_manager.go:104] Error while syncing link "1ea79daac3163e2": failed to get address from link "1ea79daac3163e2": failed to list addresses for "1ea79daac3163e2": no such device 2026-07-29T09:39:22.866383781Z I0729 09:39:22.866324 3399 cniserver.go:300] [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb network default NAD default NAD key default] DEL finished CNI request [ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 1ea79daac3163e21bb1cffba7f1427be854be9a33498e3466cb628ea2d50a4cb network default NAD default NAD key default], result "{}", err 2026-07-29T09:39:22.911956961Z I0729 09:39:22.911883 3399 pods.go:178] Deleting pod: ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 2026-07-29T09:39:22.913541443Z I0729 09:39:22.913499 3399 pods.go:238] Attempting to release IPs for pod: ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb, ips: 10.133.0.80 2026-07-29T09:39:23.416689421Z I0729 09:39:23.416634 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 2026-07-29T09:39:23.416689421Z I0729 09:39:23.416671 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-shared-b-27430b/rate-test-model-27430b-kserve-559c4784bc-qpwrb 2026-07-29T09:39:23.998793171Z I0729 09:39:23.998747 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:39:24.026030466Z I0729 09:39:24.025984 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:24.026077353Z I0729 09:39:24.026044 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:24.026077353Z I0729 09:39:24.026058 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:24.028676333Z I0729 09:39:24.028653 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressFirewall" totalItems=9 2026-07-29T09:39:24.148977509Z I0729 09:39:24.148923 3399 base_network_controller_pods.go:490] [default/opendatahub/test-curl-7783-139681293118464] creating logical port opendatahub_test-curl-7783-139681293118464 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:39:24.149109368Z I0729 09:39:24.149089 3399 kube.go:234] Patching annotations on pod opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:24.157738401Z I0729 09:39:24.157377 3399 base_network_controller_pods.go:976] [default/opendatahub/test-curl-7783-139681293118464] addLogicalPort annotation time took 8.3303ms 2026-07-29T09:39:24.159184386Z I0729 09:39:24.159154 3399 pods.go:273] [opendatahub/test-curl-7783-139681293118464] addLogicalPort took 10.305357ms, libovsdb time 1.193374ms 2026-07-29T09:39:24.521136183Z I0729 09:39:24.521092 3399 cniserver.go:311] [opendatahub/test-curl-7783-139681293118464 e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:39:24.531339692Z I0729 09:39:24.531295 3399 helper_linux.go:533] ConfigureOVS: namespace: opendatahub, podName: test-curl-7783-139681293118464, hostIfaceName: e10a02f70604d2c, network: default, NAD default, SandboxID: "e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96", PCI device ID: , UID: "690a5a69-e59c-45b8-ac2a-edabbaec99b3", MAC: 0a:58:0a:85:00:51, IPs: [10.133.0.81/23] 2026-07-29T09:39:24.592715120Z I0729 09:39:24.592660 3399 cniserver.go:300] [opendatahub/test-curl-7783-139681293118464 e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96 network default NAD default NAD key default] ADD finished CNI request [opendatahub/test-curl-7783-139681293118464 e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:53:13:d1:f4:69\",\"name\":\"e10a02f70604d2c\"},{\"mac\":\"0a:58:0a:85:00:51\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc66cdb7-8f22-45ca-a71f-9d5c2c330d99\"}],\"ips\":[{\"address\":\"10.133.0.81/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:39:27.462272436Z I0729 09:39:27.462225 3399 cniserver.go:311] [opendatahub/test-curl-7783-139681293118464 e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:39:27.486845279Z E0729 09:39:27.486804 3399 link_network_manager.go:104] Error while syncing link "e10a02f70604d2c": failed to get address from link "e10a02f70604d2c": failed to list addresses for "e10a02f70604d2c": no such device 2026-07-29T09:39:27.537868961Z I0729 09:39:27.537808 3399 cniserver.go:300] [opendatahub/test-curl-7783-139681293118464 e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96 network default NAD default NAD key default] DEL finished CNI request [opendatahub/test-curl-7783-139681293118464 e10a02f70604d2ca7610192ff72420c0a4a4e1c19240a55f6d97ed79d3b0ba96 network default NAD default NAD key default], result "{}", err 2026-07-29T09:39:27.578010679Z I0729 09:39:27.577963 3399 pods.go:178] Deleting pod: opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:27.579951575Z I0729 09:39:27.579915 3399 pods.go:238] Attempting to release IPs for pod: opendatahub/test-curl-7783-139681293118464, ips: 10.133.0.81 2026-07-29T09:39:27.579996889Z I0729 09:39:27.579975 3399 base_network_controller_policy.go:651] Processing NetworkPolicy opendatahub/opendatahub to delete 1 local pods... 2026-07-29T09:39:27.594294259Z I0729 09:39:27.594249 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:27.594329733Z I0729 09:39:27.594294 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:27.594329733Z I0729 09:39:27.594316 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod opendatahub/test-curl-7783-139681293118464 2026-07-29T09:39:28.060808646Z I0729 09:39:28.060759 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:39:29.533134587Z W0729 09:39:29.533027 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ipp-a-b9d569-openshift-default 2026-07-29T09:39:29.533134587Z W0729 09:39:29.533053 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-a-b9d569-openshift-default 2026-07-29T09:39:31.430459667Z I0729 09:39:31.430403 3399 namespace.go:89] [ai-tenant-e2e-ipp-a-b9d569] adding namespace 2026-07-29T09:39:31.431218363Z I0729 09:39:31.431180 3399 namespace.go:93] [ai-tenant-e2e-ipp-a-b9d569] adding namespace took 750.624µs 2026-07-29T09:39:31.474920970Z I0729 09:39:31.474873 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:39:31.507833026Z I0729 09:39:31.507788 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:39:32.315456647Z I0729 09:39:32.315407 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj] creating logical port openshift-ingress_payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:39:32.315572624Z I0729 09:39:32.315547 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2026-07-29T09:39:32.326335319Z I0729 09:39:32.326289 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj] addLogicalPort annotation time took 10.798994ms 2026-07-29T09:39:32.327694388Z I0729 09:39:32.327663 3399 pods.go:273] [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj] addLogicalPort took 12.270257ms, libovsdb time 843.901µs 2026-07-29T09:39:32.386441690Z I0729 09:39:32.386400 3399 base_network_controller_pods.go:490] [default/openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj] creating logical port openshift-ingress_payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:39:32.386551082Z I0729 09:39:32.386535 3399 kube.go:234] Patching annotations on pod openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 2026-07-29T09:39:32.396785001Z I0729 09:39:32.396745 3399 base_network_controller_pods.go:976] [default/openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj] addLogicalPort annotation time took 10.245996ms 2026-07-29T09:39:32.398530161Z I0729 09:39:32.398492 3399 pods.go:273] [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj] addLogicalPort took 12.111107ms, libovsdb time 1.055547ms 2026-07-29T09:39:32.441476797Z I0729 09:39:32.441430 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-ipp-a-b9d569 for network default 2026-07-29T09:39:32.441611728Z I0729 09:39:32.441579 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-ipp-a-b9d569 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:39:32.444127384Z I0729 09:39:32.444092 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-ipp-a-b9d569 resources completed, update namespace loglevel 2026-07-29T09:39:32.686816979Z I0729 09:39:32.686773 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:39:32.697242049Z I0729 09:39:32.697191 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj, hostIfaceName: 2d68a412c5b5151, network: default, NAD default, SandboxID: "2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08", PCI device ID: , UID: "cc5ebdb7-5306-4210-9f59-9b9f8b759e2c", MAC: 0a:58:0a:85:00:52, IPs: [10.133.0.82/23] 2026-07-29T09:39:32.768304369Z I0729 09:39:32.768250 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:25:8e:51:d1:a5\",\"name\":\"2d68a412c5b5151\"},{\"mac\":\"0a:58:0a:85:00:52\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9a4d99bf-56ef-4a1e-b9ef-16a67b679cd9\"}],\"ips\":[{\"address\":\"10.133.0.82/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:39:32.772365688Z I0729 09:39:32.772325 3399 cniserver.go:311] [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:39:32.784752737Z I0729 09:39:32.784700 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj, hostIfaceName: 604c54b4d711284, network: default, NAD default, SandboxID: "604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c", PCI device ID: , UID: "cace6e4d-cae8-4765-accf-ec8617ccfeb7", MAC: 0a:58:0a:85:00:53, IPs: [10.133.0.83/23] 2026-07-29T09:39:32.852159701Z I0729 09:39:32.852096 3399 cniserver.go:300] [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:a8:ea:df:48:59\",\"name\":\"604c54b4d711284\"},{\"mac\":\"0a:58:0a:85:00:53\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba7643fc-de69-4739-9c85-ea84b7744298\"}],\"ips\":[{\"address\":\"10.133.0.83/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:39:33.486097894Z W0729 09:39:33.485999 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569 2026-07-29T09:39:33.502718513Z W0729 09:39:33.502669 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ipp-a-b9d569 2026-07-29T09:39:33.548829240Z W0729 09:39:33.548785 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569 2026-07-29T09:39:45.504731718Z W0729 09:39:45.504685 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569 2026-07-29T09:39:45.519639029Z W0729 09:39:45.519590 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ipp-a-b9d569 2026-07-29T09:39:49.631747758Z W0729 09:39:49.631704 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-a-b9d569-openshift-default 2026-07-29T09:39:49.631915486Z W0729 09:39:49.631797 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ipp-a-b9d569-openshift-default 2026-07-29T09:39:52.913480975Z I0729 09:39:52.913431 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=23 2026-07-29T09:39:56.458467589Z I0729 09:39:56.458342 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:39:57.038305467Z I0729 09:39:57.038255 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod kuadrant-system/kuadrant-operator-catalog-gll82 2026-07-29T09:39:57.038349153Z I0729 09:39:57.038301 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod kuadrant-system/kuadrant-operator-catalog-gll82 2026-07-29T09:39:57.170578355Z I0729 09:39:57.170525 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:39:57.553357434Z I0729 09:39:57.553311 3399 base_network_controller_pods.go:490] [default/openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j] creating logical port openshift-ingress_e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:39:57.553506545Z I0729 09:39:57.553472 3399 kube.go:234] Patching annotations on pod openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j 2026-07-29T09:39:57.570210382Z I0729 09:39:57.570165 3399 base_network_controller_pods.go:976] [default/openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j] addLogicalPort annotation time took 16.753302ms 2026-07-29T09:39:57.572133784Z I0729 09:39:57.572099 3399 pods.go:273] [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j] addLogicalPort took 18.803413ms, libovsdb time 1.195716ms 2026-07-29T09:39:57.928374301Z I0729 09:39:57.928326 3399 cniserver.go:311] [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:39:57.939412227Z I0729 09:39:57.939358 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-ingress, podName: e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j, hostIfaceName: e423fafe2db5eee, network: default, NAD default, SandboxID: "e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9", PCI device ID: , UID: "a8eba8aa-66ea-4d87-9cfe-2ae90bb7a32d", MAC: 0a:58:0a:85:00:54, IPs: [10.133.0.84/23] 2026-07-29T09:39:58.002294194Z I0729 09:39:58.002236 3399 cniserver.go:300] [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:ec:37:c6:c2:44\",\"name\":\"e423fafe2db5eee\"},{\"mac\":\"0a:58:0a:85:00:54\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd5604e0-df0a-4cb5-b87e-2a895a7c627b\"}],\"ips\":[{\"address\":\"10.133.0.84/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:39:58.653427561Z W0729 09:39:58.653382 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:39:58.653427561Z W0729 09:39:58.653411 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:00.546616115Z I0729 09:40:00.546567 3399 namespace.go:89] [ai-tenant-e2e-ipp-b-150c68] adding namespace 2026-07-29T09:40:00.547425530Z I0729 09:40:00.547375 3399 namespace.go:93] [ai-tenant-e2e-ipp-b-150c68] adding namespace took 699.544µs 2026-07-29T09:40:00.565776946Z I0729 09:40:00.565722 3399 namespace.go:133] [ai-tenant-e2e-ipp-b-150c68] updating namespace 2026-07-29T09:40:00.582121438Z I0729 09:40:00.582080 3399 namespace.go:133] [ai-tenant-e2e-ipp-b-150c68] updating namespace 2026-07-29T09:40:00.688349326Z W0729 09:40:00.688304 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:00.688349326Z W0729 09:40:00.688327 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:01.908705480Z I0729 09:40:01.908654 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-ipp-b-150c68 for network default 2026-07-29T09:40:01.908758434Z I0729 09:40:01.908735 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-ipp-b-150c68 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:40:01.910464535Z I0729 09:40:01.910435 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-ipp-b-150c68 resources completed, update namespace loglevel 2026-07-29T09:40:02.705626158Z W0729 09:40:02.705565 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68 2026-07-29T09:40:02.720354336Z W0729 09:40:02.720310 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-ipp-b-150c68 2026-07-29T09:40:04.074921447Z W0729 09:40:04.074851 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ipp-b-150c68 2026-07-29T09:40:18.779247947Z W0729 09:40:18.779204 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:18.779329667Z W0729 09:40:18.779297 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:18.779329667Z W0729 09:40:18.779241 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:18.779352434Z W0729 09:40:18.779328 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:18.779409915Z W0729 09:40:18.779386 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:18.779409915Z W0729 09:40:18.779397 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:19.911717664Z I0729 09:40:19.911670 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=227 2026-07-29T09:40:22.912607324Z I0729 09:40:22.912556 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=53 2026-07-29T09:40:24.925680760Z W0729 09:40:24.925628 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:24.925680760Z W0729 09:40:24.925657 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-b-150c68-openshift-default 2026-07-29T09:40:25.551866246Z I0729 09:40:25.551817 3399 namespace.go:133] [ai-tenant-e2e-ipp-b-150c68] updating namespace 2026-07-29T09:40:41.259583384Z W0729 09:40:41.259529 3399 util.go:881] Failed to build global endpoints for port TCP/https/8000: empty IP address endpoints for service ai-tenant-e2e-ipp-a-b9d569/ipp-route-b9d569-kserve-workload-svc 2026-07-29T09:41:04.913445131Z I0729 09:41:04.913397 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=15 2026-07-29T09:41:22.781695593Z I0729 09:41:22.781534 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:41:24.109329616Z W0729 09:41:24.109289 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ipp-cleanup-82acc2-openshift-default 2026-07-29T09:41:24.109329616Z W0729 09:41:24.109312 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-cleanup-82acc2-openshift-default 2026-07-29T09:41:25.959648409Z I0729 09:41:25.959606 3399 namespace.go:89] [ai-tenant-e2e-ipp-cleanup-82acc2] adding namespace 2026-07-29T09:41:25.960483317Z I0729 09:41:25.960444 3399 namespace.go:93] [ai-tenant-e2e-ipp-cleanup-82acc2] adding namespace took 752.282µs 2026-07-29T09:41:25.975413300Z I0729 09:41:25.975378 3399 namespace.go:133] [ai-tenant-e2e-ipp-cleanup-82acc2] updating namespace 2026-07-29T09:41:25.989609067Z I0729 09:41:25.989569 3399 namespace.go:133] [ai-tenant-e2e-ipp-cleanup-82acc2] updating namespace 2026-07-29T09:41:26.117453943Z I0729 09:41:26.117384 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2] creating logical port odh-ai-gateway-infra_maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:41:26.117557143Z I0729 09:41:26.117537 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:26.130701877Z I0729 09:41:26.130657 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2] addLogicalPort annotation time took 13.158378ms 2026-07-29T09:41:26.132370204Z I0729 09:41:26.132331 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2] addLogicalPort took 14.960017ms, libovsdb time 1.029447ms 2026-07-29T09:41:26.967549637Z I0729 09:41:26.967492 3399 base_network_controller_policy.go:1190] Adding network policy openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2 for network default 2026-07-29T09:41:26.967688610Z I0729 09:41:26.967659 3399 base_network_controller_policy.go:1085] Policy openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2 added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-ingress,},}_LS{ML:{gateway.istio.io/managed: istio.io-gateway-controller,},} default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-monitoring,},}_LS{}_LNM default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-user-workload-monitoring,},}_LS{}_LNM] 2026-07-29T09:41:26.969857562Z I0729 09:41:26.969827 3399 base_network_controller_policy.go:1243] Create network policy openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2 resources completed, update namespace loglevel 2026-07-29T09:41:27.093110626Z I0729 09:41:27.093065 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:41:27.105327081Z I0729 09:41:27.105270 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2, hostIfaceName: 2d14ba176f12026, network: default, NAD default, SandboxID: "2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9", PCI device ID: , UID: "8eca8004-fd80-4d7b-a79f-3d6edb1730f3", MAC: 0a:58:0a:85:00:55, IPs: [10.133.0.85/23] 2026-07-29T09:41:27.380312248Z I0729 09:41:27.380266 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9 network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:f9:7d:c4:db:54\",\"name\":\"2d14ba176f12026\"},{\"mac\":\"0a:58:0a:85:00:55\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/180ff993-231e-4686-8a32-4f69810d4f21\"}],\"ips\":[{\"address\":\"10.133.0.85/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:41:27.501607413Z W0729 09:41:27.501559 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-pre-processing-e2e-ipp-cleanup-82acc2 2026-07-29T09:41:28.125785262Z W0729 09:41:28.125706 3399 util.go:881] Failed to build global endpoints for port TCP//9004: empty IP address endpoints for service openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2 2026-07-29T09:41:28.224331473Z W0729 09:41:28.224283 3399 util.go:881] Failed to build global endpoints for port TCP/https/8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2 2026-07-29T09:41:30.019549600Z I0729 09:41:30.019491 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressIP" totalItems=10 2026-07-29T09:41:34.235598126Z W0729 09:41:34.235546 3399 util.go:977] Failed to build node endpoints for node ip-10-0-133-190.ec2.internal port 8443: empty IP address endpoints for service odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2 2026-07-29T09:41:39.221146279Z W0729 09:41:39.221097 3399 util.go:881] Failed to build global endpoints for port TCP/https/443: empty IP address endpoints for service openshift-ingress/e2e-ipp-cleanup-82acc2-openshift-default 2026-07-29T09:41:39.221146279Z W0729 09:41:39.221124 3399 util.go:881] Failed to build global endpoints for port TCP/status-port/15021: empty IP address endpoints for service openshift-ingress/e2e-ipp-cleanup-82acc2-openshift-default 2026-07-29T09:41:50.946436829Z I0729 09:41:50.946397 3399 namespace.go:133] [ai-tenant-e2e-ipp-cleanup-82acc2] updating namespace 2026-07-29T09:41:51.684534503Z I0729 09:41:51.684490 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:41:51.684720885Z I0729 09:41:51.684695 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:51.696438218Z I0729 09:41:51.696398 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5] addLogicalPort annotation time took 11.78961ms 2026-07-29T09:41:51.697874070Z I0729 09:41:51.697836 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5] addLogicalPort took 13.369824ms, libovsdb time 818.246µs 2026-07-29T09:41:52.053049832Z I0729 09:41:52.053000 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:41:52.063164721Z I0729 09:41:52.063115 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5, hostIfaceName: 03c88f225640b41, network: default, NAD default, SandboxID: "03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7", PCI device ID: , UID: "24204c52-b9e5-4a33-8568-affeb4dbdbc1", MAC: 0a:58:0a:85:00:56, IPs: [10.133.0.86/23] 2026-07-29T09:41:52.121500032Z I0729 09:41:52.121441 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7 network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:d5:ad:d1:1c:65\",\"name\":\"03c88f225640b41\"},{\"mac\":\"0a:58:0a:85:00:56\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/376e15b6-75bb-47de-a608-12dacb50978b\"}],\"ips\":[{\"address\":\"10.133.0.86/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:41:54.411264006Z I0729 09:41:54.411213 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:41:54.434646672Z E0729 09:41:54.434607 3399 link_network_manager.go:104] Error while syncing link "03c88f225640b41": failed to get address from link "03c88f225640b41": failed to list addresses for "03c88f225640b41": no such device 2026-07-29T09:41:54.491265831Z I0729 09:41:54.491212 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7 network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 03c88f225640b416cd6b6496121fc7ea4d4c6b7bd9be299367586ababd824eb7 network default NAD default NAD key default], result "{}", err 2026-07-29T09:41:54.532553547Z I0729 09:41:54.532508 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:54.534620164Z I0729 09:41:54.534584 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5, ips: 10.133.0.86 2026-07-29T09:41:54.534649262Z I0729 09:41:54.534629 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:41:54.534858881Z I0729 09:41:54.534843 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:41:54.535112654Z I0729 09:41:54.535081 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:41:54.535435713Z I0729 09:41:54.535416 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:41:54.535650008Z I0729 09:41:54.535627 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:41:54.535846833Z I0729 09:41:54.535828 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:41:57.093612262Z I0729 09:41:57.093508 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2 2026-07-29T09:41:57.093612262Z I0729 09:41:57.093536 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2 2026-07-29T09:41:57.138809563Z I0729 09:41:57.138763 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:41:57.154200051Z I0729 09:41:57.154146 3399 namespace.go:133] [ai-tenant-e2e-ipp-cleanup-82acc2] updating namespace 2026-07-29T09:41:57.164184135Z E0729 09:41:57.164146 3399 link_network_manager.go:104] Error while syncing link "2d14ba176f12026": failed to get address from link "2d14ba176f12026": failed to list addresses for "2d14ba176f12026": no such device 2026-07-29T09:41:57.201253458Z I0729 09:41:57.201206 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201328060Z I0729 09:41:57.201307 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201341731Z I0729 09:41:57.201327 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201341731Z I0729 09:41:57.201334 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201351560Z I0729 09:41:57.201343 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201360991Z I0729 09:41:57.201352 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201370112Z I0729 09:41:57.201361 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.201391063Z I0729 09:41:57.201376 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-cleanup-82acc2-qp2l5 2026-07-29T09:41:57.223324844Z I0729 09:41:57.223278 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9 network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2d14ba176f12026fbaebcd2c9b946033e1bfefddedbb0dc8dd6454a93fbf93d9 network default NAD default NAD key default], result "{}", err 2026-07-29T09:41:57.279167341Z I0729 09:41:57.279122 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.281199073Z I0729 09:41:57.281165 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2, ips: 10.133.0.85 2026-07-29T09:41:57.281228681Z I0729 09:41:57.281215 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:41:57.281487138Z I0729 09:41:57.281459 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:41:57.281691035Z I0729 09:41:57.281671 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:41:57.281885142Z I0729 09:41:57.281866 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:41:57.282110951Z I0729 09:41:57.282090 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:41:57.412846140Z I0729 09:41:57.412783 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.412846140Z I0729 09:41:57.412811 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.412846140Z I0729 09:41:57.412820 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.412846140Z I0729 09:41:57.412828 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.412846140Z I0729 09:41:57.412837 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.412882653Z I0729 09:41:57.412848 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.412882653Z I0729 09:41:57.412860 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-ipp-cleanup-82acc2-6f7f9bf5d4-zh5j2 2026-07-29T09:41:57.515465269Z I0729 09:41:57.515414 3399 namespace.go:133] [ai-tenant-e2e-ipp-cleanup-82acc2] updating namespace 2026-07-29T09:41:57.639379603Z I0729 09:41:57.639327 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-ipp-cleanup-82acc2-5bdb84f8d4-vrxds 2026-07-29T09:41:57.639379603Z I0729 09:41:57.639364 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-ipp-cleanup-82acc2-5bdb84f8d4-vrxds 2026-07-29T09:41:58.310872148Z I0729 09:41:58.310822 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2-59f7fb46fd-65kbz 2026-07-29T09:41:58.310935120Z I0729 09:41:58.310885 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-ipp-cleanup-82acc2-59f7fb46fd-65kbz 2026-07-29T09:42:03.343389669Z I0729 09:42:03.343334 3399 namespace.go:133] [ai-tenant-e2e-ipp-cleanup-82acc2] updating namespace 2026-07-29T09:42:03.354057652Z I0729 09:42:03.354014 3399 namespace.go:290] [ai-tenant-e2e-ipp-cleanup-82acc2] deleting namespace 2026-07-29T09:42:03.835471512Z I0729 09:42:03.835424 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:42:07.128819794Z I0729 09:42:07.128777 3399 base_network_controller_pods.go:490] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn] creating logical port odh-ai-gateway-infra_maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:42:07.129071161Z I0729 09:42:07.129039 3399 kube.go:234] Patching annotations on pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:07.138593861Z I0729 09:42:07.138553 3399 base_network_controller_pods.go:976] [default/odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn] addLogicalPort annotation time took 9.563882ms 2026-07-29T09:42:07.140199679Z I0729 09:42:07.140164 3399 pods.go:273] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn] addLogicalPort took 11.409999ms, libovsdb time 921.96µs 2026-07-29T09:42:07.502212424Z I0729 09:42:07.502163 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:42:07.512163116Z I0729 09:42:07.512114 3399 helper_linux.go:533] ConfigureOVS: namespace: odh-ai-gateway-infra, podName: maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn, hostIfaceName: 6723989a7caa382, network: default, NAD default, SandboxID: "6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908", PCI device ID: , UID: "16ce978a-5305-4425-8321-23e798c23520", MAC: 0a:58:0a:85:00:57, IPs: [10.133.0.87/23] 2026-07-29T09:42:07.781861181Z I0729 09:42:07.781804 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908 network default NAD default NAD key default] ADD finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:73:a1:c9:ee:5d\",\"name\":\"6723989a7caa382\"},{\"mac\":\"0a:58:0a:85:00:57\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/299b957f-16b6-44c7-ad0e-0f28b844b408\"}],\"ips\":[{\"address\":\"10.133.0.87/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:42:09.377294657Z I0729 09:42:09.377244 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-ipp-cleanup-82acc2-openshift-default-6f57f7c8b6-8ds8v 2026-07-29T09:42:09.377294657Z I0729 09:42:09.377279 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-ipp-cleanup-82acc2-openshift-default-6f57f7c8b6-8ds8v 2026-07-29T09:42:10.509030873Z I0729 09:42:10.508986 3399 cniserver.go:311] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:42:10.529341545Z E0729 09:42:10.529277 3399 link_network_manager.go:104] Error while syncing link "6723989a7caa382": failed to get address from link "6723989a7caa382": failed to list addresses for "6723989a7caa382": no such device 2026-07-29T09:42:10.593935368Z I0729 09:42:10.593870 3399 cniserver.go:300] [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908 network default NAD default NAD key default] DEL finished CNI request [odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 6723989a7caa3828974f162eeed0af06caafe8bbf86ad2667b33db329a132908 network default NAD default NAD key default], result "{}", err 2026-07-29T09:42:10.639048799Z I0729 09:42:10.639003 3399 pods.go:178] Deleting pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:10.640928002Z I0729 09:42:10.640878 3399 pods.go:238] Attempting to release IPs for pod: odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn, ips: 10.133.0.87 2026-07-29T09:42:10.640962148Z I0729 09:42:10.640945 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:42:10.641178745Z I0729 09:42:10.641156 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:42:10.641361939Z I0729 09:42:10.641341 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:42:10.641612271Z I0729 09:42:10.641585 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:42:10.641917284Z I0729 09:42:10.641866 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:42:10.642172583Z I0729 09:42:10.642132 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:42:14.053750483Z I0729 09:42:14.053702 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.ClusterUserDefinedNetwork" totalItems=10 2026-07-29T09:42:22.729219195Z I0729 09:42:22.729179 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-ipp-a-b9d569 2026-07-29T09:42:22.729276442Z I0729 09:42:22.729264 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-ipp-a-b9d569 2026-07-29T09:42:22.799495667Z I0729 09:42:22.799451 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:42:22.845273574Z I0729 09:42:22.845225 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845402261Z I0729 09:42:22.845388 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845434989Z I0729 09:42:22.845424 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845464110Z I0729 09:42:22.845452 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845490605Z I0729 09:42:22.845481 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845517689Z I0729 09:42:22.845509 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845545256Z I0729 09:42:22.845536 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.845580174Z I0729 09:42:22.845571 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-a-b9d569-69vwn 2026-07-29T09:42:22.849027783Z I0729 09:42:22.848974 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:42:22.877918213Z I0729 09:42:22.877852 3399 cniserver.go:311] [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:42:22.878266242Z E0729 09:42:22.878237 3399 link_network_manager.go:104] Error while syncing link "2d68a412c5b5151": failed to get address from link "2d68a412c5b5151": failed to list addresses for "2d68a412c5b5151": no such device 2026-07-29T09:42:22.878500406Z E0729 09:42:22.878481 3399 link_network_manager.go:104] Error while syncing link "2d68a412c5b5151": failed to get address from link "2d68a412c5b5151": failed to list addresses for "2d68a412c5b5151": no such device 2026-07-29T09:42:22.910951015Z I0729 09:42:22.910885 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:42:22.910951015Z I0729 09:42:22.910943 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:42:22.910985091Z I0729 09:42:22.910958 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:42:22.910985091Z I0729 09:42:22.910970 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:42:22.910985091Z I0729 09:42:22.910979 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:42:22.930985355Z E0729 09:42:22.930943 3399 link_network_manager.go:104] Error while syncing link "604c54b4d711284": failed to get address from link "604c54b4d711284": failed to list addresses for "604c54b4d711284": no such device 2026-07-29T09:42:22.942130975Z I0729 09:42:22.942082 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2d68a412c5b515195266bbc1ce5c88ce8d99fd13b10916b0df9747927f5a7a08 network default NAD default NAD key default], result "{}", err 2026-07-29T09:42:22.987067044Z I0729 09:42:22.987017 3399 cniserver.go:300] [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 604c54b4d711284e73fcf72da767fcf9fcce3db81ab29e1e77a51ca9b669998c network default NAD default NAD key default], result "{}", err 2026-07-29T09:42:22.991678599Z I0729 09:42:22.991637 3399 pods.go:178] Deleting pod: openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2026-07-29T09:42:22.993500913Z I0729 09:42:22.993474 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj, ips: 10.133.0.82 2026-07-29T09:42:23.031993556Z I0729 09:42:23.031955 3399 pods.go:178] Deleting pod: openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 2026-07-29T09:42:23.033941292Z I0729 09:42:23.033885 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj, ips: 10.133.0.83 2026-07-29T09:42:23.173413637Z I0729 09:42:23.173373 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:42:23.456359809Z I0729 09:42:23.456309 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.456463546Z I0729 09:42:23.456418 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.456463546Z I0729 09:42:23.456432 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.456463546Z I0729 09:42:23.456440 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.456463546Z I0729 09:42:23.456446 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.456463546Z I0729 09:42:23.456454 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.456491419Z I0729 09:42:23.456472 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-ipp-a-b9d569-5d44bd5785-8z2c2 2026-07-29T09:42:23.585533734Z I0729 09:42:23.585489 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2026-07-29T09:42:23.585671059Z I0729 09:42:23.585650 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-ipp-a-b9d569-6d67dfcc6-mzrlj 2026-07-29T09:42:23.601215889Z I0729 09:42:23.601170 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 2026-07-29T09:42:23.601290058Z I0729 09:42:23.601276 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-ipp-a-b9d569-5f84788fbd-fbsmj 2026-07-29T09:42:28.221351606Z I0729 09:42:28.221303 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:42:43.869477917Z I0729 09:42:43.869426 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod ai-tenant-e2e-ipp-a-b9d569/ipp-route-b9d569-kserve-644dfc64f9-bljs9 2026-07-29T09:42:43.869477917Z I0729 09:42:43.869454 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod ai-tenant-e2e-ipp-a-b9d569/ipp-route-b9d569-kserve-644dfc64f9-bljs9 2026-07-29T09:43:04.057067709Z I0729 09:43:04.057020 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:129" type="*v1.UserDefinedNetwork" totalItems=8 2026-07-29T09:43:05.419097731Z I0729 09:43:05.419057 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:43:07.564481912Z W0729 09:43:07.564424 3399 duration.go:464] Config duration recorder: measurement expired for pod/ai-tenant-e2e-mt-2dc45aaa/e2e-lifecycle-model-2dc45aaa-kserve-6b49d88647-4phqk 2026-07-29T09:43:11.122550915Z I0729 09:43:11.122491 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:43:25.544465981Z I0729 09:43:25.544406 3399 namespace.go:133] [ai-tenant-e2e-ipp-a-b9d569] updating namespace 2026-07-29T09:43:25.554869577Z I0729 09:43:25.554826 3399 namespace.go:290] [ai-tenant-e2e-ipp-a-b9d569] deleting namespace 2026-07-29T09:43:26.068910800Z I0729 09:43:26.068842 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:43:31.224792000Z I0729 09:43:31.224747 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:43:31.224792000Z I0729 09:43:31.224783 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:43:31.224842051Z I0729 09:43:31.224794 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:43:31.224842051Z I0729 09:43:31.224807 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:43:31.224842051Z I0729 09:43:31.224816 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:43:31.224842051Z I0729 09:43:31.224824 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:43:31.781363097Z I0729 09:43:31.781307 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-ipp-a-b9d569-openshift-default-77b7fbdbd4-ssgpk 2026-07-29T09:43:31.781363097Z I0729 09:43:31.781336 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-ipp-a-b9d569-openshift-default-77b7fbdbd4-ssgpk 2026-07-29T09:43:36.936584851Z I0729 09:43:36.936535 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-ipp-b-150c68 2026-07-29T09:43:36.936584851Z I0729 09:43:36.936562 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-ipp-b-150c68 2026-07-29T09:43:36.998294109Z I0729 09:43:36.998245 3399 namespace.go:133] [ai-tenant-e2e-ipp-b-150c68] updating namespace 2026-07-29T09:43:37.039588495Z I0729 09:43:37.039538 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039588495Z I0729 09:43:37.039563 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039588495Z I0729 09:43:37.039570 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039588495Z I0729 09:43:37.039576 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039588495Z I0729 09:43:37.039582 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039637855Z I0729 09:43:37.039588 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039637855Z I0729 09:43:37.039594 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.039637855Z I0729 09:43:37.039609 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-ipp-b-150c68-9gc97 2026-07-29T09:43:37.102709312Z I0729 09:43:37.102656 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:43:37.102709312Z I0729 09:43:37.102681 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:43:37.102709312Z I0729 09:43:37.102691 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:43:37.102709312Z I0729 09:43:37.102702 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:43:37.102752924Z I0729 09:43:37.102715 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:43:37.393579865Z I0729 09:43:37.393532 3399 namespace.go:133] [ai-tenant-e2e-ipp-b-150c68] updating namespace 2026-07-29T09:43:37.806753024Z I0729 09:43:37.806700 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.806753024Z I0729 09:43:37.806725 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.806753024Z I0729 09:43:37.806731 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.806753024Z I0729 09:43:37.806736 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.806753024Z I0729 09:43:37.806742 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.806801033Z I0729 09:43:37.806749 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.806801033Z I0729 09:43:37.806763 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-ipp-b-150c68-7fbbcf8764-g246z 2026-07-29T09:43:37.821308372Z I0729 09:43:37.821253 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-ipp-b-150c68-9c8b7dbdd-d9d76 2026-07-29T09:43:37.821308372Z I0729 09:43:37.821281 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-ipp-b-150c68-9c8b7dbdd-d9d76 2026-07-29T09:43:38.140503539Z I0729 09:43:38.140453 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-ipp-b-150c68-654b964bf8-74bpd 2026-07-29T09:43:38.140545157Z I0729 09:43:38.140498 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-ipp-b-150c68-654b964bf8-74bpd 2026-07-29T09:43:41.014086987Z I0729 09:43:41.014031 3399 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 2026-07-29T09:43:43.136479595Z I0729 09:43:43.136390 3399 namespace.go:133] [ai-tenant-e2e-ipp-b-150c68] updating namespace 2026-07-29T09:43:43.148119661Z I0729 09:43:43.148080 3399 namespace.go:290] [ai-tenant-e2e-ipp-b-150c68] deleting namespace 2026-07-29T09:43:43.629677794Z I0729 09:43:43.629624 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:43:46.979703663Z I0729 09:43:46.979656 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:43:46.979802037Z I0729 09:43:46.979769 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:43:46.979802037Z I0729 09:43:46.979788 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:43:46.979827788Z I0729 09:43:46.979801 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:43:46.979827788Z I0729 09:43:46.979811 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:43:46.979827788Z I0729 09:43:46.979824 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:43:48.576476168Z I0729 09:43:48.576429 3399 cniserver.go:311] [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:43:48.612936320Z E0729 09:43:48.612874 3399 link_network_manager.go:104] Error while syncing link "e423fafe2db5eee": failed to get address from link "e423fafe2db5eee": failed to list addresses for "e423fafe2db5eee": no such device 2026-07-29T09:43:48.656468607Z I0729 09:43:48.656417 3399 cniserver.go:300] [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j e423fafe2db5eee32192ac40050628eb5d33610a4ff3f401cd54ceab7abb76e9 network default NAD default NAD key default], result "{}", err 2026-07-29T09:43:48.710859523Z I0729 09:43:48.710812 3399 pods.go:178] Deleting pod: openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j 2026-07-29T09:43:48.712720655Z I0729 09:43:48.712684 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j, ips: 10.133.0.84 2026-07-29T09:43:49.112930148Z I0729 09:43:49.112857 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j 2026-07-29T09:43:49.112930148Z I0729 09:43:49.112910 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-ipp-b-150c68-openshift-default-74c6fbb978-6pb8j 2026-07-29T09:43:54.427449695Z I0729 09:43:54.427406 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427524790Z I0729 09:43:54.427489 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427524790Z I0729 09:43:54.427504 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427543806Z I0729 09:43:54.427522 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427543806Z I0729 09:43:54.427530 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427543806Z I0729 09:43:54.427538 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427552723Z I0729 09:43:54.427546 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.427577648Z I0729 09:43:54.427565 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-a-0abb48-29755290-mzcpg 2026-07-29T09:43:54.501911739Z I0729 09:43:54.501853 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-shared-a-0abb48 2026-07-29T09:43:54.501911739Z I0729 09:43:54.501873 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-shared-a-0abb48 2026-07-29T09:43:54.560741836Z I0729 09:43:54.560702 3399 namespace.go:133] [ai-tenant-e2e-shared-a-0abb48] updating namespace 2026-07-29T09:43:54.609401126Z I0729 09:43:54.609357 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609401126Z I0729 09:43:54.609384 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609401126Z I0729 09:43:54.609389 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609401126Z I0729 09:43:54.609396 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609440440Z I0729 09:43:54.609402 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609440440Z I0729 09:43:54.609408 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609440440Z I0729 09:43:54.609414 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.609440440Z I0729 09:43:54.609427 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-a-0abb48-rrg72 2026-07-29T09:43:54.757790899Z I0729 09:43:54.757743 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:43:54.757850493Z I0729 09:43:54.757816 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:43:54.757850493Z I0729 09:43:54.757831 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:43:54.757850493Z I0729 09:43:54.757843 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:43:54.757865674Z I0729 09:43:54.757857 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:43:54.913003122Z I0729 09:43:54.912958 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-shared-a-0abb48-86b4bbf57b-8lj7t 2026-07-29T09:43:54.913003122Z I0729 09:43:54.912986 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-shared-a-0abb48-86b4bbf57b-8lj7t 2026-07-29T09:43:54.928511461Z I0729 09:43:54.928470 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.928511461Z I0729 09:43:54.928497 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.928511461Z I0729 09:43:54.928505 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.928554746Z I0729 09:43:54.928513 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.928554746Z I0729 09:43:54.928521 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.928554746Z I0729 09:43:54.928528 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.928554746Z I0729 09:43:54.928537 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-shared-a-0abb48-59f5597cc5-rwvm9 2026-07-29T09:43:54.941366110Z I0729 09:43:54.941328 3399 namespace.go:133] [ai-tenant-e2e-shared-a-0abb48] updating namespace 2026-07-29T09:43:54.946814618Z I0729 09:43:54.946776 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-shared-a-0abb48-c8f7d9d46-vrwrx 2026-07-29T09:43:54.946814618Z I0729 09:43:54.946806 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-shared-a-0abb48-c8f7d9d46-vrwrx 2026-07-29T09:43:59.032221595Z I0729 09:43:59.032162 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressQoS" totalItems=9 2026-07-29T09:44:00.909368681Z I0729 09:44:00.909328 3399 namespace.go:133] [ai-tenant-e2e-shared-a-0abb48] updating namespace 2026-07-29T09:44:00.919620132Z I0729 09:44:00.919569 3399 namespace.go:290] [ai-tenant-e2e-shared-a-0abb48] deleting namespace 2026-07-29T09:44:01.392652226Z I0729 09:44:01.392602 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:44:04.084835827Z I0729 09:44:04.084797 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:44:04.084931785Z I0729 09:44:04.084913 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:44:04.084962835Z I0729 09:44:04.084933 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:44:04.084962835Z I0729 09:44:04.084946 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:44:04.084974071Z I0729 09:44:04.084957 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:44:04.084983427Z I0729 09:44:04.084975 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:44:06.970437898Z I0729 09:44:06.970382 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-shared-a-0abb48-openshift-default-5f945bb56d-qjj6g 2026-07-29T09:44:06.970437898Z I0729 09:44:06.970420 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-shared-a-0abb48-openshift-default-5f945bb56d-qjj6g 2026-07-29T09:44:07.038760073Z I0729 09:44:07.038712 3399 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:129" type="*v1.EgressService" totalItems=6 2026-07-29T09:44:12.019343331Z I0729 09:44:12.019301 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019383202Z I0729 09:44:12.019338 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019383202Z I0729 09:44:12.019348 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019383202Z I0729 09:44:12.019357 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019383202Z I0729 09:44:12.019365 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019383202Z I0729 09:44:12.019372 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019383202Z I0729 09:44:12.019379 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.019412712Z I0729 09:44:12.019400 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-key-cleanup-e2e-shared-b-27430b-29755290-62trf 2026-07-29T09:44:12.112424901Z I0729 09:44:12.112391 3399 base_network_controller_policy.go:1302] Deleting network policy openshift-ingress/payload-processing-e2e-shared-b-27430b 2026-07-29T09:44:12.112424901Z I0729 09:44:12.112409 3399 base_network_controller_policy.go:1338] Cleaning up network policy openshift-ingress/payload-processing-e2e-shared-b-27430b 2026-07-29T09:44:12.173531902Z I0729 09:44:12.173484 3399 namespace.go:133] [ai-tenant-e2e-shared-b-27430b] updating namespace 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215287 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215317 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215326 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215334 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215342 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215351 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215361 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.215547144Z I0729 09:44:12.215374 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-revoke-keys-e2e-shared-b-27430b-xhrxc 2026-07-29T09:44:12.220862661Z I0729 09:44:12.220826 3399 cniserver.go:311] [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:44:12.257503518Z I0729 09:44:12.257464 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:44:12.257564140Z I0729 09:44:12.257553 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:44:12.257596572Z I0729 09:44:12.257587 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:44:12.257627194Z I0729 09:44:12.257619 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:44:12.257658105Z I0729 09:44:12.257649 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:44:12.263364668Z E0729 09:44:12.263184 3399 link_network_manager.go:104] Error while syncing link "dc25acd3edc06df": failed to get address from link "dc25acd3edc06df": failed to list addresses for "dc25acd3edc06df": no such device 2026-07-29T09:44:12.263364668Z E0729 09:44:12.263288 3399 link_network_manager.go:104] Error while syncing link "dc25acd3edc06df": failed to get address from link "dc25acd3edc06df": failed to list addresses for "dc25acd3edc06df": no such device 2026-07-29T09:44:12.308724818Z I0729 09:44:12.308679 3399 cniserver.go:300] [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b dc25acd3edc06df67b187c51f9086f38848a6fcc601b0c45fb54c9f495540c70 network default NAD default NAD key default], result "{}", err 2026-07-29T09:44:12.354515199Z I0729 09:44:12.354465 3399 pods.go:178] Deleting pod: openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b 2026-07-29T09:44:12.356358648Z I0729 09:44:12.356294 3399 pods.go:238] Attempting to release IPs for pod: openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b, ips: 10.133.0.73 2026-07-29T09:44:12.555516859Z I0729 09:44:12.555456 3399 namespace.go:133] [ai-tenant-e2e-shared-b-27430b] updating namespace 2026-07-29T09:44:13.012404652Z I0729 09:44:13.012360 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-pre-processing-e2e-shared-b-27430b-656494ff6-xscpj 2026-07-29T09:44:13.012404652Z I0729 09:44:13.012391 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-pre-processing-e2e-shared-b-27430b-656494ff6-xscpj 2026-07-29T09:44:13.023768887Z I0729 09:44:13.023735 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.023768887Z I0729 09:44:13.023760 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.023804550Z I0729 09:44:13.023769 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.023804550Z I0729 09:44:13.023776 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.023804550Z I0729 09:44:13.023783 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.023804550Z I0729 09:44:13.023791 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.023814710Z I0729 09:44:13.023805 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-e2e-shared-b-27430b-79f8d6999f-qt26p 2026-07-29T09:44:13.251565414Z I0729 09:44:13.251521 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b 2026-07-29T09:44:13.251634735Z I0729 09:44:13.251615 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/payload-processing-e2e-shared-b-27430b-98c6c7f8b-fp78b 2026-07-29T09:44:18.535730531Z I0729 09:44:18.535687 3399 namespace.go:133] [ai-tenant-e2e-shared-b-27430b] updating namespace 2026-07-29T09:44:18.547377438Z I0729 09:44:18.547334 3399 namespace.go:290] [ai-tenant-e2e-shared-b-27430b] deleting namespace 2026-07-29T09:44:19.026585441Z I0729 09:44:19.026537 3399 namespace.go:133] [odh-ai-gateway-infra] updating namespace 2026-07-29T09:44:25.068564769Z I0729 09:44:25.068512 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/e2e-shared-b-27430b-openshift-default-fbb5f548d-tsdlh 2026-07-29T09:44:25.068611957Z I0729 09:44:25.068581 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/e2e-shared-b-27430b-openshift-default-fbb5f548d-tsdlh 2026-07-29T09:44:35.811764745Z I0729 09:44:35.811716 3399 namespace.go:89] [openshift-must-gather-dcgtc] adding namespace 2026-07-29T09:44:35.812444838Z I0729 09:44:35.812408 3399 namespace.go:93] [openshift-must-gather-dcgtc] adding namespace took 664.336µs 2026-07-29T09:44:35.833575696Z I0729 09:44:35.833499 3399 namespace.go:133] [openshift-must-gather-dcgtc] updating namespace 2026-07-29T09:44:35.855365244Z I0729 09:44:35.855318 3399 base_network_controller_pods.go:490] [default/openshift-must-gather-dcgtc/must-gather-mnrr4] creating logical port openshift-must-gather-dcgtc_must-gather-mnrr4 for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:44:35.855547622Z I0729 09:44:35.855515 3399 kube.go:234] Patching annotations on pod openshift-must-gather-dcgtc/must-gather-mnrr4 2026-07-29T09:44:35.866488389Z I0729 09:44:35.866442 3399 base_network_controller_pods.go:976] [default/openshift-must-gather-dcgtc/must-gather-mnrr4] addLogicalPort annotation time took 10.973329ms 2026-07-29T09:44:35.867926965Z I0729 09:44:35.867879 3399 pods.go:273] [openshift-must-gather-dcgtc/must-gather-mnrr4] addLogicalPort took 12.596726ms, libovsdb time 890.734µs 2026-07-29T09:44:36.221831102Z I0729 09:44:36.221785 3399 cniserver.go:311] [openshift-must-gather-dcgtc/must-gather-mnrr4 60ece57fed8d1554bdab55e2d928805eead245e4d4c9b42a6d1f278cb70b8004 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:44:36.231796513Z I0729 09:44:36.231744 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-must-gather-dcgtc, podName: must-gather-mnrr4, hostIfaceName: 60ece57fed8d155, network: default, NAD default, SandboxID: "60ece57fed8d1554bdab55e2d928805eead245e4d4c9b42a6d1f278cb70b8004", PCI device ID: , UID: "581240f5-5f98-4253-96bd-100ee667f12a", MAC: 0a:58:0a:85:00:58, IPs: [10.133.0.88/23] 2026-07-29T09:44:36.292330685Z I0729 09:44:36.292283 3399 cniserver.go:300] [openshift-must-gather-dcgtc/must-gather-mnrr4 60ece57fed8d1554bdab55e2d928805eead245e4d4c9b42a6d1f278cb70b8004 network default NAD default NAD key default] ADD finished CNI request [openshift-must-gather-dcgtc/must-gather-mnrr4 60ece57fed8d1554bdab55e2d928805eead245e4d4c9b42a6d1f278cb70b8004 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:95:41:b9:9e:80\",\"name\":\"60ece57fed8d155\"},{\"mac\":\"0a:58:0a:85:00:58\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1873499a-6663-4333-8323-b9688cbb7015\"}],\"ips\":[{\"address\":\"10.133.0.88/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:44:48.996883706Z I0729 09:44:48.996824 3399 base_network_controller_pods.go:490] [default/openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh] creating logical port openshift-must-gather-dcgtc_perf-node-gather-daemonset-jwmkh for pod on switch ip-10-0-133-190.ec2.internal 2026-07-29T09:44:48.998717204Z I0729 09:44:48.998668 3399 kube.go:234] Patching annotations on pod openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh 2026-07-29T09:44:49.011209213Z I0729 09:44:49.011164 3399 base_network_controller_pods.go:976] [default/openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh] addLogicalPort annotation time took 12.557153ms 2026-07-29T09:44:49.013733549Z I0729 09:44:49.013664 3399 pods.go:273] [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh] addLogicalPort took 16.889526ms, libovsdb time 1.083183ms 2026-07-29T09:44:49.402682159Z I0729 09:44:49.402623 3399 cniserver.go:311] [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184 network default NAD default NAD key default] ADD starting CNI request 2026-07-29T09:44:49.416440173Z I0729 09:44:49.416391 3399 helper_linux.go:533] ConfigureOVS: namespace: openshift-must-gather-dcgtc, podName: perf-node-gather-daemonset-jwmkh, hostIfaceName: de900d7a65b5700, network: default, NAD default, SandboxID: "de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184", PCI device ID: , UID: "c96b4038-e4be-45bb-be9a-36223f2f107b", MAC: 0a:58:0a:85:00:59, IPs: [10.133.0.89/23] 2026-07-29T09:44:49.495309227Z I0729 09:44:49.495146 3399 cniserver.go:300] [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184 network default NAD default NAD key default] ADD finished CNI request [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:13:e3:fd:b5:fe\",\"name\":\"de900d7a65b5700\"},{\"mac\":\"0a:58:0a:85:00:59\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ff27c868-f3c9-46d0-b704-046640147bf2\"}],\"ips\":[{\"address\":\"10.133.0.89/23\",\"gateway\":\"10.133.0.1\",\"interface\":1}]}", err 2026-07-29T09:44:52.912245103Z I0729 09:44:52.912200 3399 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=50 2026-07-29T09:45:03.365832477Z I0729 09:45:03.365700 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-cleanup-restrict to delete 1 local pods... 2026-07-29T09:45:03.365832477Z I0729 09:45:03.365729 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-authorino-allow to delete 1 local pods... 2026-07-29T09:45:03.365832477Z I0729 09:45:03.365741 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-deployment-ns to delete 1 local pods... 2026-07-29T09:45:03.365832477Z I0729 09:45:03.365752 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-gateway to delete 1 local pods... 2026-07-29T09:45:03.365832477Z I0729 09:45:03.365762 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-monitoring to delete 1 local pods... 2026-07-29T09:45:03.365832477Z I0729 09:45:03.365777 3399 base_network_controller_policy.go:651] Processing NetworkPolicy odh-ai-gateway-infra/maas-api-allow-cleanup to delete 1 local pods... 2026-07-29T09:45:04.277751859Z I0729 09:45:04.277704 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277843598Z I0729 09:45:04.277810 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277843598Z I0729 09:45:04.277830 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277866455Z I0729 09:45:04.277840 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277866455Z I0729 09:45:04.277846 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277866455Z I0729 09:45:04.277856 3399 obj_retry.go:798] default/netPolController: ignoring delete event for resource in terminal state *factory.localPodSelector odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277909515Z I0729 09:45:04.277866 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:04.277909515Z I0729 09:45:04.277882 3399 obj_retry.go:798] default/networkController: ignoring delete event for resource in terminal state *v1.Pod odh-ai-gateway-infra/maas-api-key-cleanup-29755260-8jz9r 2026-07-29T09:45:07.093933810Z I0729 09:45:07.093878 3399 pods.go:178] Deleting pod: openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh 2026-07-29T09:45:07.095461831Z I0729 09:45:07.095427 3399 pods.go:238] Attempting to release IPs for pod: openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh, ips: 10.133.0.89 2026-07-29T09:45:07.248983648Z I0729 09:45:07.248934 3399 cniserver.go:311] [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184 network default NAD default NAD key default] DEL starting CNI request 2026-07-29T09:45:07.275809338Z E0729 09:45:07.275771 3399 link_network_manager.go:104] Error while syncing link "de900d7a65b5700": failed to get address from link "de900d7a65b5700": failed to list addresses for "de900d7a65b5700": no such device 2026-07-29T09:45:07.328402881Z I0729 09:45:07.328354 3399 cniserver.go:300] [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184 network default NAD default NAD key default] DEL finished CNI request [openshift-must-gather-dcgtc/perf-node-gather-daemonset-jwmkh de900d7a65b570021b1a2d170d53081c391af4b97ea589d575204236e1d81184 network default NAD default NAD key default], result "{}", err