+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-68.ec2.internal ++ [[ -n ip-10-0-0-68.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-68.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + wait-and-add-garp-drop-flow br-ex + cni-bin-copy + . /host/etc/os-release + local bridge=br-ex + local max_retries=90 + local retries=0 + [[ 0 -lt 90 ]] + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260504-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260504-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260504-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ ovs-vsctl list-ports br-ex ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + return 1 + (( retries += 1 )) + [[ 1 -eq 15 ]] + sleep 2 ++ date '+%m%d %H:%M:%S.%N' + echo 'I0508 18:02:43.995953533 - disable conntrack on geneve port' I0508 18:02:43.995953533 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0508 18:02:44.015524889 - starting ovnkube-node' I0508 18:02:44.015524889 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-68.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-68.ec2.internal --init-node ip-10-0-0-68.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-68.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0508 18:02:44.060183 3653 config.go:2752] Parsed config file /run/ovnkube-config/ovnkube.conf I0508 18:02:44.060258 3653 config.go:2753] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AllowICMPNetworkPolicy:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-c68dfbf252.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName: DPUNodeLeaseRenewInterval:10 DPUNodeLeaseDuration:40} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology: FRRNamespace:}} I0508 18:02:44.062131 3653 kube.go:437] Waiting for certificate I0508 18:02:44.062168 3653 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0508 18:02:44.062194 3653 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0508 18:02:44.062918 3653 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0508 18:02:44.093719 3653 reflector.go:367] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0508 18:02:44.093738 3653 reflector.go:411] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0508 18:02:44.098044 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" logger="kubernetes.io/kube-apiserver-client" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" totalItems=2 duration="4.264712ms" I0508 18:02:44.098068 3653 reflector.go:446] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0508 18:02:44.098106 3653 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9q6dr" I0508 18:02:44.103721 3653 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9q6dr" I0508 18:02:44.103777 3653 reflector.go:373] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0508 18:02:45.062798 3653 kube.go:444] Certificate found I0508 18:02:45.063381 3653 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0508 18:02:45.064671 3653 server.go:247] Metric Server starts to listen on 127.0.0.1:29103 I0508 18:02:45.109710 3653 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-09 17:57:44 +0000 UTC" deadline="2026-05-09 11:27:49.798079449 +0000 UTC" I0508 18:02:45.110044 3653 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h25m4.688156892s" I0508 18:02:45.141619 3653 metrics.go:159] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0508 18:02:45.149562 3653 metrics.go:489] Create OVN Metrics Server on address: 127.0.0.1:29105 I0508 18:02:45.149633 3653 server.go:119] MetricServer registers OVS metrics I0508 18:02:45.149897 3653 server.go:123] MetricServer registers OVN DB metrics I0508 18:02:45.152011 3653 node_controller_manager.go:345] Starting the node network controller manager, Mode: full I0508 18:02:45.152076 3653 factory.go:600] Starting watch factory I0508 18:02:45.152198 3653 reflector.go:367] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152246 3653 reflector.go:411] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152336 3653 reflector.go:367] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152371 3653 reflector.go:411] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152426 3653 reflector.go:367] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152468 3653 reflector.go:411] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152863 3653 reflector.go:367] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.152881 3653 reflector.go:411] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.153027 3653 reflector.go:367] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.153087 3653 reflector.go:411] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.153063 3653 reflector.go:367] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.153368 3653 reflector.go:411] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.154710 3653 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0508 18:02:45.154745 3653 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0508 18:02:45.154910 3653 controller_manager.go:389] Starting the ovnkube controller I0508 18:02:45.154920 3653 controller_manager.go:394] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-68.ec2.internal I0508 18:02:45.154994 3653 controller_manager.go:414] NBDB zone sync took: 67.294µs I0508 18:02:45.155003 3653 factory.go:600] Starting watch factory I0508 18:02:45.168093 3653 ovn_db.go:388] Found OVN NB DB: &{0x1de2de0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0508 18:02:45.168111 3653 ovn_db.go:395] Found OVN SB DB: &{0x1de30a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0508 18:02:45.171953 3653 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0508 18:02:45.171987 3653 server.go:127] MetricServer registers OVN Controller metrics I0508 18:02:45.172012 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=3 duration="18.603022ms" I0508 18:02:45.172034 3653 reflector.go:446] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.174970 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=30 duration="22.569799ms" I0508 18:02:45.175027 3653 reflector.go:446] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.176304 3653 server.go:131] MetricServer registers OVN Northd metrics I0508 18:02:45.178056 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=36 duration="24.9369ms" I0508 18:02:45.178100 3653 reflector.go:446] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.179625 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=80 duration="26.714501ms" I0508 18:02:45.179722 3653 reflector.go:446] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.180511 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=45 duration="28.220163ms" I0508 18:02:45.180631 3653 reflector.go:446] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.182264 3653 metrics.go:496] OVN Metrics Server starts to run ... I0508 18:02:45.182356 3653 server.go:247] Metric Server starts to listen on 127.0.0.1:29105 I0508 18:02:45.182758 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="k8s.io/client-go/informers/factory.go:161" totalItems=34 duration="30.264278ms" I0508 18:02:45.182849 3653 reflector.go:446] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:161" I0508 18:02:45.258956 3653 factory.go:2018] *v1.EndpointSlice informer cache synced successfully I0508 18:02:45.258984 3653 factory.go:2018] *v1.Pod informer cache synced successfully I0508 18:02:45.258988 3653 factory.go:2018] *v1.NetworkPolicy informer cache synced successfully I0508 18:02:45.258991 3653 factory.go:2018] *v1.Namespace informer cache synced successfully I0508 18:02:45.258994 3653 factory.go:2018] *v1.Node informer cache synced successfully I0508 18:02:45.258997 3653 factory.go:2018] *v1.Service informer cache synced successfully I0508 18:02:45.259110 3653 reflector.go:367] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0508 18:02:45.259127 3653 reflector.go:411] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0508 18:02:45.259110 3653 reflector.go:367] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0508 18:02:45.259169 3653 reflector.go:411] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0508 18:02:45.261130 3653 factory.go:2018] *v1.NetworkPolicy informer cache synced successfully I0508 18:02:45.261142 3653 factory.go:2018] *v1.Namespace informer cache synced successfully I0508 18:02:45.261145 3653 factory.go:2018] *v1.Node informer cache synced successfully I0508 18:02:45.261148 3653 factory.go:2018] *v1.Service informer cache synced successfully I0508 18:02:45.261151 3653 factory.go:2018] *v1.EndpointSlice informer cache synced successfully I0508 18:02:45.261154 3653 factory.go:2018] *v1.Pod informer cache synced successfully + [[ 1 -lt 90 ]] + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + return 1 + (( retries += 1 )) + [[ 2 -eq 15 ]] + sleep 2 I0508 18:02:46.298790 3653 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://api.kx-c68dfbf252.hypershift.local:443/apis/policy.networking.k8s.io/v1alpha1/adminnetworkpolicies?allowWatchBookmarks=true&resourceVersionMatch=NotOlderThan&sendInitialEvents=true&timeout=9m43s&timeoutSeconds=583&watch=true" I0508 18:02:46.304549 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" totalItems=1 duration="1.045393182s" I0508 18:02:46.304574 3653 reflector.go:446] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0508 18:02:46.317364 3653 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://api.kx-c68dfbf252.hypershift.local:443/apis/policy.networking.k8s.io/v1alpha1/baselineadminnetworkpolicies?allowWatchBookmarks=true&resourceVersionMatch=NotOlderThan&sendInitialEvents=true&timeout=9m16s&timeoutSeconds=556&watch=true" I0508 18:02:46.322725 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" totalItems=1 duration="1.063530032s" I0508 18:02:46.322751 3653 reflector.go:446] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0508 18:02:46.364695 3653 factory.go:2018] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0508 18:02:46.364711 3653 factory.go:2018] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0508 18:02:46.364788 3653 reflector.go:367] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:46.364800 3653 reflector.go:411] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:46.367061 3653 factory.go:2018] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0508 18:02:46.367073 3653 factory.go:2018] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0508 18:02:47.382707 3653 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://api.kx-c68dfbf252.hypershift.local:443/apis/k8s.ovn.org/v1/egressips?allowWatchBookmarks=true&resourceVersionMatch=NotOlderThan&sendInitialEvents=true&timeout=5m28s&timeoutSeconds=328&watch=true" I0508 18:02:47.387921 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="1.023096034s" I0508 18:02:47.387944 3653 reflector.go:446] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.470966 3653 factory.go:2018] *v1.EgressIP informer cache synced successfully I0508 18:02:47.471072 3653 reflector.go:367] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.471087 3653 reflector.go:411] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.473127 3653 factory.go:2018] *v1.EgressIP informer cache synced successfully I0508 18:02:47.476900 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="5.769091ms" I0508 18:02:47.476923 3653 reflector.go:446] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.478777 3653 factory.go:2018] *v1.EgressFirewall informer cache synced successfully I0508 18:02:47.478854 3653 reflector.go:367] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.478872 3653 reflector.go:411] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.483874 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="4.971779ms" I0508 18:02:47.483908 3653 reflector.go:446] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.577253 3653 factory.go:2018] *v1.EgressFirewall informer cache synced successfully I0508 18:02:47.582426 3653 factory.go:2018] *v1.EgressQoS informer cache synced successfully I0508 18:02:47.582523 3653 reflector.go:367] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.582536 3653 reflector.go:411] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.584682 3653 factory.go:2018] *v1.EgressQoS informer cache synced successfully I0508 18:02:47.589799 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="7.232203ms" I0508 18:02:47.589825 3653 reflector.go:446] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.688608 3653 factory.go:2018] *v1.EgressService informer cache synced successfully I0508 18:02:47.688738 3653 reflector.go:367] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.688754 3653 reflector.go:411] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:47.690801 3653 factory.go:2018] *v1.EgressService informer cache synced successfully + [[ 2 -lt 90 ]] + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + return 1 + (( retries += 1 )) + [[ 3 -eq 15 ]] + sleep 2 I0508 18:02:48.707369 3653 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://api.kx-c68dfbf252.hypershift.local:443/apis/k8s.ovn.org/v1/adminpolicybasedexternalroutes?allowWatchBookmarks=true&resourceVersionMatch=NotOlderThan&sendInitialEvents=true&timeout=8m57s&timeoutSeconds=537&watch=true" I0508 18:02:48.712987 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="1.02420155s" I0508 18:02:48.713014 3653 reflector.go:446] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:48.794997 3653 factory.go:2018] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0508 18:02:48.795198 3653 reflector.go:367] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0508 18:02:48.795229 3653 reflector.go:411] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0508 18:02:48.796515 3653 factory.go:2018] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0508 18:02:48.800856 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" totalItems=2 duration="5.599702ms" I0508 18:02:48.800883 3653 reflector.go:446] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0508 18:02:48.801749 3653 factory.go:2018] *v1.NetworkAttachmentDefinition informer cache synced successfully I0508 18:02:48.801826 3653 reflector.go:367] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:48.801841 3653 reflector.go:411] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:48.801893 3653 reflector.go:367] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:48.801908 3653 reflector.go:411] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:48.807069 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="5.137288ms" I0508 18:02:48.807089 3653 reflector.go:446] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:48.901568 3653 factory.go:2018] *v1.NetworkAttachmentDefinition informer cache synced successfully I0508 18:02:49.846900 3653 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://api.kx-c68dfbf252.hypershift.local:443/apis/k8s.ovn.org/v1/clusteruserdefinednetworks?allowWatchBookmarks=true&resourceVersionMatch=NotOlderThan&sendInitialEvents=true&timeout=7m6s&timeoutSeconds=426&watch=true" I0508 18:02:49.852174 3653 reflector.go:978] "Exiting watch because received the bookmark that marks the end of initial events stream" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" totalItems=1 duration="1.050285291s" I0508 18:02:49.852200 3653 reflector.go:446] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" I0508 18:02:49.907115 3653 factory.go:2018] *v1.UserDefinedNetwork informer cache synced successfully I0508 18:02:49.907134 3653 factory.go:2018] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0508 18:02:49.907142 3653 factory.go:719] Watch Factory start up complete, took: 4.755068463s I0508 18:02:49.907237 3653 default_node_network_controller.go:191] Enable node proxy healthz server on 0.0.0.0:10256 I0508 18:02:49.908543 3653 factory.go:2018] *v1.UserDefinedNetwork informer cache synced successfully I0508 18:02:49.908555 3653 factory.go:2018] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0508 18:02:49.908562 3653 factory.go:719] Watch Factory start up complete, took: 4.753557648s I0508 18:02:49.908569 3653 controller_manager.go:425] Waiting up to 5m0s for a node to have "ip-10-0-0-68.ec2.internal" zone I0508 18:02:49.951369 3653 default_node_network_controller.go:739] Initializing the default node network controller + [[ 3 -lt 90 ]] + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + return 1 + (( retries += 1 )) + [[ 4 -eq 15 ]] + sleep 2 I0508 18:02:50.038251 3653 udn_isolation.go:101] Starting UDN host isolation manager I0508 18:02:50.044438 3653 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0508 18:02:50.099584 3653 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0508 18:02:50.099670 3653 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0508 18:02:50.099688 3653 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0508 18:02:50.133816 3653 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0508 18:02:50.133881 3653 default_node_network_controller.go:854] Node ip-10-0-0-68.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0508 18:02:50.139851 3653 kube.go:265] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.68"] k8s.ovn.org/zone-name:ip-10-0-0-68.ec2.internal] on node ip-10-0-0-68.ec2.internal I0508 18:02:50.158945 3653 controller_manager.go:448] Waiting for node in zone sync took: 250.361452ms I0508 18:02:50.163770 3653 controller_manager.go:641] Switching to transit router for layer2 networks I0508 18:02:50.163783 3653 kube.go:265] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-68.ec2.internal I0508 18:02:50.164268 3653 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0508 18:02:50.171085 3653 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0508 18:02:50.171279 3653 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0508 18:02:50.171408 3653 gateway_init.go:263] Preparing Gateway I0508 18:02:50.171420 3653 gateway_shared_intf.go:1682] Creating new gateway I0508 18:02:50.177938 3653 metrics.go:159] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0508 18:02:50.178616 3653 addressset_manager.go:127] Starting pod-selector-address-set-manager controller I0508 18:02:50.178646 3653 controller.go:133] Adding controller pod-selector-address-set-manager-pod event handlers I0508 18:02:50.178681 3653 controller.go:133] Adding controller pod-selector-address-set-manager-namespace event handlers I0508 18:02:50.178700 3653 shared_informer.go:349] "Waiting for caches to sync" controller="pod-selector-address-set-manager-pod" I0508 18:02:50.178711 3653 shared_informer.go:356] "Caches are synced" controller="pod-selector-address-set-manager-pod" I0508 18:02:50.178759 3653 shared_informer.go:349] "Waiting for caches to sync" controller="pod-selector-address-set-manager-namespace" I0508 18:02:50.178776 3653 shared_informer.go:356] "Caches are synced" controller="pod-selector-address-set-manager-namespace" I0508 18:02:50.178800 3653 controller.go:157] Starting controller pod-selector-address-set-manager-pod with 1 workers I0508 18:02:50.178814 3653 controller.go:157] Starting controller pod-selector-address-set-manager-namespace with 1 workers I0508 18:02:50.178824 3653 controller.go:157] Starting controller pod-selector-address-set-manager-addrset with 1 workers I0508 18:02:50.178939 3653 services_controller.go:69] Creating services controller for network=default I0508 18:02:50.179049 3653 udn_enabled_svc.go:80] Waiting for service informer to sync I0508 18:02:50.179075 3653 controller.go:133] Adding controller udn-node-topology-node event handlers I0508 18:02:50.179100 3653 shared_informer.go:349] "Waiting for caches to sync" controller="udn-node-topology-node" I0508 18:02:50.179115 3653 shared_informer.go:356] "Caches are synced" controller="udn-node-topology-node" I0508 18:02:50.179122 3653 controller.go:157] Starting controller udn-node-topology-node with 15 workers I0508 18:02:50.179147 3653 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0508 18:02:50.179170 3653 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0508 18:02:50.179181 3653 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0508 18:02:50.179191 3653 nad_controller.go:540] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0508 18:02:50.179384 3653 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0508 18:02:50.179425 3653 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0508 18:02:50.179467 3653 nad_controller.go:579] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 51.46µs I0508 18:02:50.179489 3653 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 54.865µs I0508 18:02:50.179497 3653 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 71.112µs I0508 18:02:50.179502 3653 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0508 18:02:50.179509 3653 nad_controller.go:378] [zone-nad-controller NAD controller]: started I0508 18:02:50.179518 3653 default_network_controller.go:325] Starting the default network controller I0508 18:02:50.179615 3653 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 13.828µs I0508 18:02:50.179751 3653 udn_enabled_svc.go:95] Performing full resync I0508 18:02:50.180149 3653 udn_enabled_svc.go:99] Waiting for handler to sync I0508 18:02:50.180165 3653 udn_enabled_svc.go:104] Starting worker I0508 18:02:50.189881 3653 default_network_controller.go:398] Cleaning External Gateway ECMP routes I0508 18:02:50.189944 3653 repair.go:33] Syncing exgw routes took 50.235µs I0508 18:02:50.189969 3653 default_network_controller.go:417] Starting all the Watchers... I0508 18:02:50.190130 3653 namespace.go:97] [openshift-logging] adding namespace I0508 18:02:50.190133 3653 namespace.go:97] [kube-system] adding namespace I0508 18:02:50.190139 3653 namespace.go:97] [openshift-kube-scheduler-operator] adding namespace I0508 18:02:50.190139 3653 namespace.go:97] [openshift-cluster-samples-operator] adding namespace I0508 18:02:50.190133 3653 namespace.go:97] [openshift-cluster-csi-drivers] adding namespace I0508 18:02:50.190152 3653 namespace.go:97] [openshift-authentication-operator] adding namespace I0508 18:02:50.190160 3653 namespace.go:97] [openshift-package-operator] adding namespace I0508 18:02:50.190164 3653 namespace.go:97] [openshift-dns] adding namespace I0508 18:02:50.190166 3653 namespace.go:97] [openshift-backplane-csm] adding namespace I0508 18:02:50.190175 3653 namespace.go:97] [openshift-cluster-version] adding namespace I0508 18:02:50.190166 3653 namespace.go:97] [openshift-controller-manager-operator] adding namespace I0508 18:02:50.190178 3653 namespace.go:97] [kube-public] adding namespace I0508 18:02:50.190183 3653 namespace.go:97] [openshift-authentication] adding namespace I0508 18:02:50.190185 3653 namespace.go:97] [openshift-insights] adding namespace I0508 18:02:50.190236 3653 namespace.go:97] [openshift-cluster-machine-approver] adding namespace I0508 18:02:50.190922 3653 namespace.go:101] [openshift-logging] adding namespace took 779.142µs I0508 18:02:50.190944 3653 namespace.go:97] [openshift-backplane-srep-ro] adding namespace I0508 18:02:50.191568 3653 namespace.go:101] [openshift-backplane-srep-ro] adding namespace took 617.162µs I0508 18:02:50.191590 3653 namespace.go:97] [openshift-kube-scheduler] adding namespace W0508 18:02:50.192304 3653 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-thlmm: pod kube-system/global-pull-secret-syncer-thlmm: no pod IPs found I0508 18:02:50.192316 3653 namespace.go:101] [openshift-kube-scheduler] adding namespace took 720.722µs W0508 18:02:50.192322 3653 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-qkh4z: pod kube-system/global-pull-secret-syncer-qkh4z: no pod IPs found I0508 18:02:50.192348 3653 namespace.go:97] [openshift-operators] adding namespace I0508 18:02:50.193056 3653 namespace.go:101] [kube-system] adding namespace took 2.905958ms I0508 18:02:50.193080 3653 namespace.go:97] [openshift-kube-controller-manager-operator] adding namespace I0508 18:02:50.193720 3653 namespace.go:101] [openshift-kube-scheduler-operator] adding namespace took 3.568047ms I0508 18:02:50.193761 3653 namespace.go:97] [openshift-cloud-network-config-controller] adding namespace I0508 18:02:50.194442 3653 namespace.go:101] [openshift-cluster-samples-operator] adding namespace took 4.290033ms I0508 18:02:50.194463 3653 namespace.go:97] [openshift-config-managed] adding namespace I0508 18:02:50.195101 3653 namespace.go:101] [openshift-cluster-csi-drivers] adding namespace took 4.942124ms I0508 18:02:50.195123 3653 namespace.go:97] [openshift-operator-lifecycle-manager] adding namespace I0508 18:02:50.195669 3653 namespace.go:101] [openshift-authentication-operator] adding namespace took 5.505703ms I0508 18:02:50.195690 3653 namespace.go:97] [openshift-backplane] adding namespace I0508 18:02:50.196802 3653 namespace.go:101] [openshift-package-operator] adding namespace took 6.633347ms I0508 18:02:50.196822 3653 namespace.go:97] [openshift-network-diagnostics] adding namespace I0508 18:02:50.197439 3653 namespace.go:101] [openshift-dns] adding namespace took 7.263426ms I0508 18:02:50.197458 3653 namespace.go:97] [openshift-node] adding namespace I0508 18:02:50.198077 3653 namespace.go:101] [openshift-backplane-csm] adding namespace took 7.898258ms I0508 18:02:50.198091 3653 namespace.go:97] [openshift-backplane-srep] adding namespace I0508 18:02:50.198662 3653 namespace.go:101] [openshift-cluster-version] adding namespace took 8.480452ms I0508 18:02:50.198679 3653 namespace.go:97] [openshift-must-gather-operator] adding namespace I0508 18:02:50.199318 3653 namespace.go:101] [openshift-controller-manager-operator] adding namespace took 9.1357ms I0508 18:02:50.199330 3653 namespace.go:97] [open-cluster-management-agent-addon] adding namespace I0508 18:02:50.199973 3653 namespace.go:101] [kube-public] adding namespace took 9.789386ms I0508 18:02:50.199984 3653 namespace.go:97] [openshift-cluster-node-tuning-operator] adding namespace I0508 18:02:50.200725 3653 namespace.go:101] [openshift-authentication] adding namespace took 10.531106ms I0508 18:02:50.200739 3653 namespace.go:97] [openshift-backplane-managed-scripts] adding namespace I0508 18:02:50.201357 3653 namespace.go:101] [openshift-insights] adding namespace took 11.16033ms I0508 18:02:50.201373 3653 namespace.go:97] [openshift-backplane-cee] adding namespace I0508 18:02:50.201943 3653 namespace.go:101] [openshift-cluster-machine-approver] adding namespace took 11.694848ms I0508 18:02:50.201958 3653 namespace.go:97] [openshift-kube-controller-manager] adding namespace I0508 18:02:50.202464 3653 namespace.go:101] [openshift-operators] adding namespace took 10.107569ms I0508 18:02:50.202478 3653 namespace.go:97] [openshift-monitoring] adding namespace I0508 18:02:50.203030 3653 namespace.go:101] [openshift-kube-controller-manager-operator] adding namespace took 9.944562ms I0508 18:02:50.203044 3653 namespace.go:97] [openshift-apiserver-operator] adding namespace I0508 18:02:50.203648 3653 namespace.go:101] [openshift-cloud-network-config-controller] adding namespace took 9.878087ms I0508 18:02:50.203664 3653 namespace.go:97] [openshift-image-registry] adding namespace I0508 18:02:50.204238 3653 namespace.go:101] [openshift-config-managed] adding namespace took 9.767954ms I0508 18:02:50.204254 3653 namespace.go:97] [openshift-route-controller-manager] adding namespace I0508 18:02:50.204833 3653 namespace.go:101] [openshift-operator-lifecycle-manager] adding namespace took 9.699285ms I0508 18:02:50.204853 3653 namespace.go:97] [openshift-backplane-mobb] adding namespace W0508 18:02:50.205341 3653 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-9xjjn: pod openshift-network-diagnostics/network-check-target-9xjjn: no pod IPs found W0508 18:02:50.205356 3653 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-crlnt: pod openshift-network-diagnostics/network-check-target-crlnt: no pod IPs found I0508 18:02:50.205388 3653 namespace.go:101] [openshift-backplane] adding namespace took 9.689758ms I0508 18:02:50.205408 3653 namespace.go:97] [openshift-machine-api] adding namespace I0508 18:02:50.205953 3653 namespace.go:101] [openshift-network-diagnostics] adding namespace took 9.120851ms I0508 18:02:50.205970 3653 namespace.go:97] [openshift-backplane-lpsre] adding namespace I0508 18:02:50.206578 3653 namespace.go:101] [openshift-node] adding namespace took 9.114156ms I0508 18:02:50.206593 3653 namespace.go:97] [openshift-infra] adding namespace I0508 18:02:50.207050 3653 namespace.go:101] [openshift-backplane-srep] adding namespace took 8.954271ms I0508 18:02:50.207064 3653 namespace.go:97] [openshift-dns-operator] adding namespace I0508 18:02:50.207644 3653 namespace.go:101] [openshift-must-gather-operator] adding namespace took 8.942092ms I0508 18:02:50.207664 3653 namespace.go:97] [openshift-service-ca-operator] adding namespace I0508 18:02:50.208200 3653 namespace.go:101] [open-cluster-management-agent-addon] adding namespace took 8.862094ms I0508 18:02:50.208236 3653 namespace.go:97] [openshift-console-operator] adding namespace I0508 18:02:50.208720 3653 namespace.go:101] [openshift-cluster-node-tuning-operator] adding namespace took 8.727175ms I0508 18:02:50.208739 3653 namespace.go:97] [openshift-kube-storage-version-migrator-operator] adding namespace I0508 18:02:50.209378 3653 namespace.go:101] [openshift-backplane-managed-scripts] adding namespace took 8.632285ms I0508 18:02:50.209395 3653 namespace.go:97] [openshift-marketplace] adding namespace I0508 18:02:50.210013 3653 namespace.go:101] [openshift-backplane-cee] adding namespace took 8.634219ms I0508 18:02:50.210029 3653 namespace.go:97] [openshift-deployment-validation-operator] adding namespace I0508 18:02:50.210627 3653 namespace.go:101] [openshift-kube-controller-manager] adding namespace took 8.663134ms I0508 18:02:50.210648 3653 namespace.go:97] [openshift-ingress] adding namespace I0508 18:02:50.211136 3653 namespace.go:101] [openshift-monitoring] adding namespace took 8.651908ms I0508 18:02:50.211712 3653 namespace.go:101] [openshift-apiserver-operator] adding namespace took 8.66226ms I0508 18:02:50.212135 3653 namespace.go:97] [openshift-controller-manager] adding namespace I0508 18:02:50.212294 3653 namespace.go:101] [openshift-image-registry] adding namespace took 8.62291ms I0508 18:02:50.212312 3653 namespace.go:97] [openshift-config-operator] adding namespace I0508 18:02:50.212862 3653 namespace.go:101] [openshift-route-controller-manager] adding namespace took 8.600363ms I0508 18:02:50.212877 3653 namespace.go:97] [openshift-user-workload-monitoring] adding namespace I0508 18:02:50.213318 3653 namespace.go:101] [openshift-backplane-mobb] adding namespace took 8.458448ms I0508 18:02:50.213333 3653 namespace.go:97] [openshift-ovn-kubernetes] adding namespace I0508 18:02:50.213807 3653 namespace.go:101] [openshift-machine-api] adding namespace took 8.392412ms I0508 18:02:50.213824 3653 namespace.go:97] [openshift-console] adding namespace I0508 18:02:50.214249 3653 namespace.go:101] [openshift-backplane-lpsre] adding namespace took 8.271667ms I0508 18:02:50.214267 3653 namespace.go:97] [openshift-cluster-storage-operator] adding namespace I0508 18:02:50.214803 3653 namespace.go:101] [openshift-infra] adding namespace took 8.204164ms I0508 18:02:50.214822 3653 namespace.go:97] [openshift-backplane-tam] adding namespace I0508 18:02:50.215352 3653 namespace.go:101] [openshift-dns-operator] adding namespace took 8.280006ms I0508 18:02:50.215372 3653 namespace.go:97] [kube-node-lease] adding namespace I0508 18:02:50.215889 3653 namespace.go:101] [openshift-service-ca-operator] adding namespace took 8.219627ms I0508 18:02:50.215908 3653 namespace.go:97] [open-cluster-management-2q5t7gpgv0osum5rqnh4kbqvj22icgps] adding namespace I0508 18:02:50.216477 3653 namespace.go:101] [openshift-console-operator] adding namespace took 8.232347ms I0508 18:02:50.216498 3653 namespace.go:97] [openshift-cloud-controller-manager] adding namespace I0508 18:02:50.216978 3653 namespace.go:101] [openshift-kube-storage-version-migrator-operator] adding namespace took 8.232321ms I0508 18:02:50.216995 3653 namespace.go:97] [openshift-host-network] adding namespace I0508 18:02:50.217529 3653 namespace.go:101] [openshift-marketplace] adding namespace took 8.127938ms I0508 18:02:50.217546 3653 namespace.go:97] [dedicated-admin] adding namespace I0508 18:02:50.218166 3653 namespace.go:101] [openshift-deployment-validation-operator] adding namespace took 8.130808ms I0508 18:02:50.218184 3653 namespace.go:97] [openshift-config] adding namespace I0508 18:02:50.218743 3653 namespace.go:101] [openshift-ingress] adding namespace took 8.087977ms I0508 18:02:50.218757 3653 namespace.go:97] [openshift-network-node-identity] adding namespace I0508 18:02:50.219207 3653 namespace.go:101] [openshift-controller-manager] adding namespace took 7.06071ms I0508 18:02:50.219239 3653 namespace.go:97] [openshift-backplane-mcs-tier-two] adding namespace I0508 18:02:50.219807 3653 namespace.go:101] [openshift-config-operator] adding namespace took 7.489517ms I0508 18:02:50.219821 3653 namespace.go:97] [openshift-cloud-credential-operator] adding namespace I0508 18:02:50.220364 3653 namespace.go:101] [openshift-user-workload-monitoring] adding namespace took 7.464368ms I0508 18:02:50.220383 3653 namespace.go:97] [openshift-kube-apiserver-operator] adding namespace I0508 18:02:50.220868 3653 namespace.go:101] [openshift-ovn-kubernetes] adding namespace took 7.526429ms I0508 18:02:50.220882 3653 namespace.go:97] [openshift-machine-config-operator] adding namespace I0508 18:02:50.221474 3653 namespace.go:101] [openshift-console] adding namespace took 7.644323ms I0508 18:02:50.221487 3653 namespace.go:97] [openshift-console-user-settings] adding namespace I0508 18:02:50.222047 3653 namespace.go:101] [openshift-cluster-storage-operator] adding namespace took 7.774037ms I0508 18:02:50.222063 3653 namespace.go:97] [openshift-network-operator] adding namespace I0508 18:02:50.222610 3653 namespace.go:101] [openshift-backplane-tam] adding namespace took 7.781591ms I0508 18:02:50.222624 3653 namespace.go:97] [openshift-multus] adding namespace I0508 18:02:50.223165 3653 namespace.go:101] [kube-node-lease] adding namespace took 7.785121ms I0508 18:02:50.223184 3653 namespace.go:97] [openshift] adding namespace I0508 18:02:50.223719 3653 namespace.go:101] [open-cluster-management-2q5t7gpgv0osum5rqnh4kbqvj22icgps] adding namespace took 7.803944ms I0508 18:02:50.223733 3653 namespace.go:97] [openshift-kube-apiserver] adding namespace I0508 18:02:50.224368 3653 namespace.go:101] [openshift-cloud-controller-manager] adding namespace took 7.86267ms I0508 18:02:50.224387 3653 namespace.go:97] [openshift-ingress-canary] adding namespace I0508 18:02:50.225070 3653 namespace.go:101] [openshift-host-network] adding namespace took 8.067217ms I0508 18:02:50.225085 3653 namespace.go:97] [openshift-network-console] adding namespace I0508 18:02:50.225611 3653 namespace.go:101] [dedicated-admin] adding namespace took 8.057465ms I0508 18:02:50.225631 3653 namespace.go:97] [openshift-apiserver] adding namespace I0508 18:02:50.226058 3653 namespace.go:101] [openshift-config] adding namespace took 7.86818ms I0508 18:02:50.226076 3653 namespace.go:97] [openshift-operators-redhat] adding namespace I0508 18:02:50.226651 3653 namespace.go:101] [openshift-network-node-identity] adding namespace took 7.888079ms I0508 18:02:50.226666 3653 namespace.go:97] [openshift-customer-monitoring] adding namespace I0508 18:02:50.227331 3653 namespace.go:101] [openshift-backplane-mcs-tier-two] adding namespace took 8.084232ms I0508 18:02:50.227837 3653 namespace.go:101] [openshift-cloud-credential-operator] adding namespace took 8.00809ms I0508 18:02:50.228522 3653 namespace.go:101] [openshift-kube-apiserver-operator] adding namespace took 8.132567ms I0508 18:02:50.228974 3653 namespace.go:101] [openshift-machine-config-operator] adding namespace took 8.086822ms I0508 18:02:50.228993 3653 namespace.go:97] [openshift-backplane-cse] adding namespace I0508 18:02:50.229521 3653 namespace.go:101] [openshift-console-user-settings] adding namespace took 8.028523ms W0508 18:02:50.230079 3653 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-rdmhw: pod openshift-multus/network-metrics-daemon-rdmhw: no pod IPs found W0508 18:02:50.230091 3653 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-cmgxx: pod openshift-multus/network-metrics-daemon-cmgxx: no pod IPs found I0508 18:02:50.230128 3653 namespace.go:101] [openshift-network-operator] adding namespace took 8.056793ms I0508 18:02:50.230152 3653 namespace.go:97] [default] adding namespace I0508 18:02:50.230915 3653 namespace.go:101] [openshift-multus] adding namespace took 8.284451ms I0508 18:02:50.231467 3653 namespace.go:101] [openshift] adding namespace took 8.275534ms I0508 18:02:50.232084 3653 namespace.go:101] [openshift-kube-apiserver] adding namespace took 8.345604ms I0508 18:02:50.232795 3653 namespace.go:101] [openshift-ingress-canary] adding namespace took 8.39949ms I0508 18:02:50.232818 3653 namespace.go:97] [openshift-ingress-operator] adding namespace I0508 18:02:50.233474 3653 namespace.go:101] [openshift-network-console] adding namespace took 8.38245ms I0508 18:02:50.234023 3653 namespace.go:101] [openshift-apiserver] adding namespace took 8.383402ms I0508 18:02:50.234657 3653 namespace.go:101] [openshift-operators-redhat] adding namespace took 8.574465ms I0508 18:02:50.234679 3653 namespace.go:97] [openshift-etcd] adding namespace I0508 18:02:50.235232 3653 namespace.go:101] [openshift-customer-monitoring] adding namespace took 8.559233ms I0508 18:02:50.235750 3653 namespace.go:101] [openshift-backplane-cse] adding namespace took 6.752507ms I0508 18:02:50.236371 3653 namespace.go:101] [default] adding namespace took 6.211973ms I0508 18:02:50.237037 3653 namespace.go:101] [openshift-ingress-operator] adding namespace took 4.214746ms I0508 18:02:50.237549 3653 namespace.go:101] [openshift-etcd] adding namespace took 2.864768ms W0508 18:02:50.237644 3653 master.go:468] Unable to parse local node ip-10-0-0-68.ec2.internal chassis-id annotation. Chassis may be removed during sync W0508 18:02:50.237653 3653 master.go:481] Unable to parse remote node ip-10-0-2-114.ec2.internal chassis-id annotation. Chassis may be removed during sync I0508 18:02:50.237660 3653 master.go:491] Removing stale chassis with ID/Name: 3c72c779-15bc-4f27-830d-1e0b8ccb5691, hostname: ip-10-0-0-68.ec2.internal I0508 18:02:50.238083 3653 master.go:558] Adding or Updating local node "ip-10-0-0-68.ec2.internal" for network "default" I0508 18:02:50.238154 3653 obj_retry.go:605] default/networkController: failed to create *v1.Node ip-10-0-2-114.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-114.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-114.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-114.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-114.ec2.internal I0508 18:02:50.242024 3653 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-68.ec2.internal for the network default I0508 18:02:50.242366 3653 zone_ic_handler.go:195] Time taken to create transit switch: 327.461µs I0508 18:02:50.243029 3653 default_network_controller.go:829] Node add failed for ip-10-0-0-68.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-68.ec2.internal, error getting gateway config for node ip-10-0-0-68.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-68.ec2.internal", failed to update chassis to local for local node ip-10-0-0-68.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-68.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-68.ec2.internal] E0508 18:02:50.243072 3653 obj_retry.go:602] default/networkController: failed to create *v1.Node ip-10-0-0-68.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-68.ec2.internal, error getting gateway config for node ip-10-0-0-68.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-68.ec2.internal", failed to update chassis to local for local node ip-10-0-0-68.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-68.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-68.ec2.internal] I0508 18:02:50.243120 3653 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0508 18:02:50.243140 3653 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0508 18:02:50.243156 3653 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0508 18:02:50.243136 3653 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-68.ec2.internal", UID:"12317a11-ff5a-4910-83b1-a4c0b313c107", APIVersion:"v1", ResourceVersion:"8539", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-68.ec2.internal, error getting gateway config for node ip-10-0-0-68.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-68.ec2.internal", failed to update chassis to local for local node ip-10-0-0-68.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-68.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-68.ec2.internal] I0508 18:02:50.243169 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-68.ec2.internal in network "default" I0508 18:02:50.243228 3653 node_tracker.go:235] Node ip-10-0-0-68.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-68.ec2.internal" I0508 18:02:50.243250 3653 node_tracker.go:169] Node ip-10-0-0-68.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:50.243261 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:50.243272 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-114.ec2.internal in network "default" I0508 18:02:50.243285 3653 node_tracker.go:235] Node ip-10-0-2-114.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-114.ec2.internal" I0508 18:02:50.243294 3653 node_tracker.go:169] Node ip-10-0-2-114.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:50.243300 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:50.285492 3653 iptables.go:108] Creating table: filter chain: FORWARD I0508 18:02:50.292261 3653 iptables.go:108] Creating table: filter chain: OUTPUT I0508 18:02:50.298708 3653 gateway_shared_intf.go:1810] Gateway Creation Complete I0508 18:02:50.298727 3653 kube.go:265] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-68.ec2.internal","mac-address":"12:fd:e2:1e:6c:53","ip-addresses":["10.0.0.68/24"],"ip-address":"10.0.0.68/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3c72c779-15bc-4f27-830d-1e0b8ccb5691 k8s.ovn.org/node-encap-ips:["10.0.0.68"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.68/24"} k8s.ovn.org/zone-name:ip-10-0-0-68.ec2.internal] on node ip-10-0-0-68.ec2.internal I0508 18:02:50.314940 3653 master.go:558] Adding or Updating local node "ip-10-0-0-68.ec2.internal" for network "default" I0508 18:02:50.314970 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-68.ec2.internal in network "default" I0508 18:02:50.315158 3653 node_tracker.go:169] Node ip-10-0-0-68.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:50.315176 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:50.319807 3653 default_node_network_controller.go:548] Node connection status = connected W0508 18:02:50.320312 3653 gateway.go:216] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-rdmhw: pod openshift-multus/network-metrics-daemon-rdmhw: no pod IPs found W0508 18:02:50.320328 3653 gateway.go:216] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-9xjjn: pod openshift-network-diagnostics/network-check-target-9xjjn: no pod IPs found W0508 18:02:50.320337 3653 gateway.go:216] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-thlmm: pod kube-system/global-pull-secret-syncer-thlmm: no pod IPs found I0508 18:02:50.322782 3653 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-68.ec2.internal for the network default I0508 18:02:50.322987 3653 zone_ic_handler.go:195] Time taken to create transit switch: 191.102µs I0508 18:02:50.343942 3653 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0508 18:02:50.343963 3653 services_controller.go:208] Setting up event handlers for services for network=default I0508 18:02:50.344023 3653 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0508 18:02:50.344087 3653 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0508 18:02:50.344103 3653 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0508 18:02:50.444632 3653 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0508 18:02:50.444658 3653 repair.go:60] Starting repairing loop for services I0508 18:02:50.444785 3653 repair.go:131] Deleted 0 stale service LBs I0508 18:02:50.444811 3653 repair.go:137] Deleted 0 stale Chassis Template Vars I0508 18:02:50.444834 3653 repair.go:62] Finished repairing loop for services: 177.793µs I0508 18:02:50.444873 3653 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0508 18:02:50.444887 3653 services_controller.go:254] Starting workers for network=default I0508 18:02:50.445259 3653 obj_retry.go:605] default/networkController: failed to create *v1.Pod openshift-network-diagnostics/network-check-target-crlnt, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-crlnt: suppressed error logged: pod openshift-network-diagnostics/network-check-target-crlnt: no pod IPs found I0508 18:02:50.445269 3653 base_network_controller_pods.go:489] [default/openshift-network-diagnostics/network-check-target-9xjjn] creating logical port openshift-network-diagnostics_network-check-target-9xjjn for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:50.445302 3653 obj_retry.go:605] default/networkController: failed to create *v1.Pod kube-system/global-pull-secret-syncer-qkh4z, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-qkh4z: suppressed error logged: pod kube-system/global-pull-secret-syncer-qkh4z: no pod IPs found I0508 18:02:50.445307 3653 base_network_controller_pods.go:489] [default/openshift-multus/network-metrics-daemon-rdmhw] creating logical port openshift-multus_network-metrics-daemon-rdmhw for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:50.445363 3653 base_network_controller_pods.go:489] [default/kube-system/global-pull-secret-syncer-thlmm] creating logical port kube-system_global-pull-secret-syncer-thlmm for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:50.445467 3653 kube.go:233] Patching annotations on pod openshift-network-diagnostics/network-check-target-9xjjn I0508 18:02:50.445546 3653 kube.go:233] Patching annotations on pod kube-system/global-pull-secret-syncer-thlmm I0508 18:02:50.445619 3653 kube.go:233] Patching annotations on pod openshift-multus/network-metrics-daemon-rdmhw I0508 18:02:50.464135 3653 base_network_controller_pods.go:974] [default/openshift-network-diagnostics/network-check-target-9xjjn] addLogicalPort annotation time took 18.77143ms I0508 18:02:50.465288 3653 pods.go:270] [openshift-network-diagnostics/network-check-target-9xjjn] addLogicalPort took 20.030744ms, libovsdb time 898.65µs I0508 18:02:50.467578 3653 base_network_controller_pods.go:974] [default/openshift-multus/network-metrics-daemon-rdmhw] addLogicalPort annotation time took 22.195484ms I0508 18:02:50.468489 3653 pods.go:270] [openshift-multus/network-metrics-daemon-rdmhw] addLogicalPort took 23.189935ms, libovsdb time 687.174µs I0508 18:02:50.471436 3653 base_network_controller_pods.go:974] [default/kube-system/global-pull-secret-syncer-thlmm] addLogicalPort annotation time took 26.009325ms I0508 18:02:50.472203 3653 pods.go:270] [kube-system/global-pull-secret-syncer-thlmm] addLogicalPort took 26.84653ms, libovsdb time 574.724µs I0508 18:02:50.472273 3653 obj_retry.go:605] default/networkController: failed to create *v1.Pod openshift-multus/network-metrics-daemon-cmgxx, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-cmgxx: suppressed error logged: pod openshift-multus/network-metrics-daemon-cmgxx: no pod IPs found I0508 18:02:50.472659 3653 admin_network_policy_controller.go:227] Starting controller default-network-controller I0508 18:02:50.472684 3653 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0508 18:02:50.472696 3653 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0508 18:02:50.472700 3653 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0508 18:02:50.472925 3653 repair.go:29] Repairing admin network policies took 219.268µs I0508 18:02:50.473042 3653 repair.go:92] Repairing baseline admin network policies took 106.506µs I0508 18:02:50.473447 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0508 18:02:50.473934 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0508 18:02:50.476483 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0508 18:02:50.476511 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0508 18:02:50.476524 3653 base_network_controller_policy.go:1102] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0508 18:02:50.478367 3653 base_network_controller_policy.go:1280] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0508 18:02:50.478395 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0508 18:02:50.478402 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0508 18:02:50.478779 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0508 18:02:50.478802 3653 base_network_controller_policy.go:1227] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0508 18:02:50.478811 3653 base_network_controller_policy.go:1102] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0508 18:02:50.480545 3653 base_network_controller_policy.go:1280] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0508 18:02:50.480563 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-version/default-deny for network default I0508 18:02:50.480570 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-version/default-deny added to peer address sets [] I0508 18:02:50.482028 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0508 18:02:50.482056 3653 base_network_controller_policy.go:1227] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0508 18:02:50.482068 3653 base_network_controller_policy.go:1102] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0508 18:02:50.484812 3653 base_network_controller_policy.go:1280] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0508 18:02:50.484837 3653 base_network_controller_policy.go:1227] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0508 18:02:50.484849 3653 base_network_controller_policy.go:1102] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0508 18:02:50.486034 3653 base_network_controller_policy.go:1280] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0508 18:02:50.486053 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0508 18:02:50.486078 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [default-network-controller_LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0508 18:02:50.488453 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0508 18:02:50.488480 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0508 18:02:50.488494 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0508 18:02:50.489139 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0508 18:02:50.489169 3653 base_network_controller_policy.go:1227] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0508 18:02:50.489181 3653 base_network_controller_policy.go:1102] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0508 18:02:50.489976 3653 base_network_controller_policy.go:1280] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0508 18:02:50.490002 3653 base_network_controller_policy.go:1227] Adding network policy openshift-marketplace/default-deny-all for network default I0508 18:02:50.490011 3653 base_network_controller_policy.go:1102] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0508 18:02:50.491656 3653 base_network_controller_policy.go:1280] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0508 18:02:50.491677 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0508 18:02:50.491686 3653 base_network_controller_policy.go:1102] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0508 18:02:50.492358 3653 base_network_controller_policy.go:1280] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0508 18:02:50.492381 3653 base_network_controller_policy.go:1227] Adding network policy openshift-marketplace/marketplace-operator for network default I0508 18:02:50.492396 3653 base_network_controller_policy.go:1102] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0508 18:02:50.493850 3653 base_network_controller_policy.go:1280] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0508 18:02:50.493875 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0508 18:02:50.493887 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0508 18:02:50.494649 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0508 18:02:50.494675 3653 base_network_controller_policy.go:1227] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0508 18:02:50.494691 3653 base_network_controller_policy.go:1102] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0508 18:02:50.495875 3653 base_network_controller_policy.go:1280] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0508 18:02:50.495900 3653 base_network_controller_policy.go:1227] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0508 18:02:50.495915 3653 base_network_controller_policy.go:1102] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0508 18:02:50.497104 3653 base_network_controller_policy.go:1280] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0508 18:02:50.497122 3653 base_network_controller_policy.go:1227] Adding network policy openshift-operators/default-allow-all for network default I0508 18:02:50.497132 3653 base_network_controller_policy.go:1102] Policy openshift-operators/default-allow-all added to peer address sets [] I0508 18:02:50.499436 3653 base_network_controller_policy.go:1280] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0508 18:02:50.499461 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0508 18:02:50.499471 3653 base_network_controller_policy.go:1102] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0508 18:02:50.500041 3653 base_network_controller_policy.go:1280] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0508 18:02:50.500066 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0508 18:02:50.500074 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0508 18:02:50.500725 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0508 18:02:50.500743 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0508 18:02:50.500752 3653 base_network_controller_policy.go:1102] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0508 18:02:50.501564 3653 base_network_controller_policy.go:1280] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0508 18:02:50.501590 3653 base_network_controller_policy.go:1227] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0508 18:02:50.501603 3653 base_network_controller_policy.go:1102] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0508 18:02:50.503044 3653 base_network_controller_policy.go:1280] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0508 18:02:50.503064 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0508 18:02:50.503076 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0508 18:02:50.503738 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0508 18:02:50.503832 3653 base_network_controller_policy.go:1227] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0508 18:02:50.503845 3653 base_network_controller_policy.go:1102] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0508 18:02:50.505193 3653 base_network_controller_policy.go:1280] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0508 18:02:50.505233 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0508 18:02:50.505245 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0508 18:02:50.505821 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0508 18:02:50.505845 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0508 18:02:50.505856 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0508 18:02:50.506520 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0508 18:02:50.506543 3653 base_network_controller_policy.go:1227] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0508 18:02:50.506549 3653 base_network_controller_policy.go:1102] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0508 18:02:50.506866 3653 base_network_controller_policy.go:1280] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0508 18:02:50.506885 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0508 18:02:50.506894 3653 base_network_controller_policy.go:1102] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0508 18:02:50.507405 3653 base_network_controller_policy.go:1280] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0508 18:02:50.507423 3653 base_network_controller_policy.go:1227] Adding network policy openshift-marketplace/unpack-bundles for network default I0508 18:02:50.507432 3653 base_network_controller_policy.go:1102] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0508 18:02:50.508476 3653 base_network_controller_policy.go:1280] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0508 18:02:50.508501 3653 base_network_controller_policy.go:1227] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0508 18:02:50.508516 3653 base_network_controller_policy.go:1102] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0508 18:02:50.508830 3653 base_network_controller_policy.go:1280] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0508 18:02:50.508858 3653 controller.go:157] Starting controller default-network-controller-egressip-nad with 1 workers I0508 18:02:50.513772 3653 egressfirewall.go:367] Starting EgressFirewall controller I0508 18:02:50.513807 3653 controller.go:133] Adding controller egress-firewall-controller event handlers I0508 18:02:50.513830 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0508 18:02:50.513846 3653 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0508 18:02:50.513858 3653 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0508 18:02:50.513884 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0508 18:02:50.513895 3653 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0508 18:02:50.514111 3653 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0508 18:02:50.514125 3653 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0508 18:02:50.514133 3653 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0508 18:02:50.514142 3653 egressqos.go:184] Setting up event handlers for EgressQoS I0508 18:02:50.514239 3653 egressfirewall.go:708] Syncing node "ip-10-0-0-68.ec2.internal" for egress firewall I0508 18:02:50.514257 3653 egressfirewall.go:708] Syncing node "ip-10-0-2-114.ec2.internal" for egress firewall I0508 18:02:50.514688 3653 egressqos.go:236] Starting EgressQoS Controller I0508 18:02:50.514702 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0508 18:02:50.514709 3653 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0508 18:02:50.514714 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0508 18:02:50.514718 3653 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0508 18:02:50.514723 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0508 18:02:50.514727 3653 shared_informer.go:356] "Caches are synced" controller="egressqos" I0508 18:02:50.514731 3653 egressqos.go:250] Repairing EgressQoSes I0508 18:02:50.514734 3653 egressqos.go:390] Starting repairing loop for egressqos I0508 18:02:50.514841 3653 egressqos.go:392] Finished repairing loop for egressqos: 105.357µs I0508 18:02:50.514862 3653 egressservice_zone.go:132] Setting up event handlers for Egress Services I0508 18:02:50.514900 3653 egressqos.go:995] Processing sync for EgressQoS node ip-10-0-0-68.ec2.internal I0508 18:02:50.514952 3653 egressqos.go:998] Finished syncing EgressQoS node ip-10-0-0-68.ec2.internal : 54.471µs I0508 18:02:50.514991 3653 egressservice_zone.go:210] Starting Egress Services Controller I0508 18:02:50.515005 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0508 18:02:50.515013 3653 shared_informer.go:356] "Caches are synced" controller="egressservices" I0508 18:02:50.515020 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0508 18:02:50.515038 3653 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0508 18:02:50.515048 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0508 18:02:50.515056 3653 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0508 18:02:50.515063 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0508 18:02:50.515074 3653 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0508 18:02:50.515078 3653 egressservice_zone.go:228] Repairing Egress Services I0508 18:02:50.515413 3653 master_controller.go:88] Starting Admin Policy Based Route Controller I0508 18:02:50.515426 3653 external_controller.go:278] Starting Admin Policy Based Route Controller I0508 18:02:50.515473 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-68.ec2.internal I0508 18:02:50.515484 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-68.ec2.internal: 11.797µs I0508 18:02:50.515491 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-114.ec2.internal I0508 18:02:50.515495 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-114.ec2.internal: 4.112µs I0508 18:02:50.515556 3653 default_network_controller.go:591] Completing all the Watchers took 325.57691ms I0508 18:02:50.515577 3653 default_network_controller.go:595] Starting unidling controllers I0508 18:02:50.515587 3653 unidle.go:46] Registering OVN SB ControllerEvent handler I0508 18:02:50.515597 3653 unidle.go:63] Populating Initial ContollerEvent events I0508 18:02:50.515614 3653 unidle.go:79] Setting up event handlers for services I0508 18:02:50.515676 3653 ovnkube.go:555] Waiting for OVN northbound database changes to be processed by ovn-controller I0508 18:02:50.521696 3653 ovnkube.go:560] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0508 18:02:50.820196 3653 default_node_network_controller.go:548] Node connection status = connected I0508 18:02:50.838411 3653 bridgeconfig.go:537] Gateway is ready I0508 18:02:50.838430 3653 gateway_shared_intf.go:1728] Creating Gateway Openflow Manager I0508 18:02:50.851334 3653 node_ip_handler_linux.go:310] Node primary address changed to 10.0.0.68. Updating OVN encap IP. I0508 18:02:50.856987 3653 node_ip_handler_linux.go:631] Will not update encap IP 10.0.0.68 - it is already configured I0508 18:02:50.857005 3653 node_ip_handler_linux.go:573] Node address changed to map[10.0.0.68/24:{} 172.20.0.1/32:{}]. Updating annotations. I0508 18:02:50.857262 3653 kube.go:265] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.68/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-68.ec2.internal","mac-address":"12:fd:e2:1e:6c:53","ip-addresses":["10.0.0.68/24"],"ip-address":"10.0.0.68/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3c72c779-15bc-4f27-830d-1e0b8ccb5691 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.68/24"}] on node ip-10-0-0-68.ec2.internal I0508 18:02:50.872207 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-68.ec2.internal in network "default" I0508 18:02:50.872275 3653 node_tracker.go:169] Node ip-10-0-0-68.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:50.872284 3653 master.go:558] Adding or Updating local node "ip-10-0-0-68.ec2.internal" for network "default" I0508 18:02:50.872292 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:50.872331 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-68.ec2.internal I0508 18:02:50.872350 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-68.ec2.internal: 26.58µs I0508 18:02:50.872363 3653 egressfirewall.go:708] Syncing node "ip-10-0-0-68.ec2.internal" for egress firewall I0508 18:02:50.872993 3653 gateway_shared_intf.go:2017] Setting OVN Masquerade route with source: 10.0.0.68 I0508 18:02:50.873181 3653 kube.go:265] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-68.ec2.internal I0508 18:02:50.877391 3653 default_network_controller.go:1091] Egress IP detected IP address change for node ip-10-0-0-68.ec2.internal. Updating no re-route policies I0508 18:02:50.877541 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-rftck I0508 18:02:50.877550 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-t72ff I0508 18:02:50.877559 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-rftck I0508 18:02:50.877567 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-t72ff I0508 18:02:50.877560 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-68.ec2.internal I0508 18:02:50.877580 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-98dqp I0508 18:02:50.877588 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-b4xcr I0508 18:02:50.877593 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-thlmm I0508 18:02:50.877595 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-network-operator/iptables-alerter-98dqp I0508 18:02:50.877597 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-b4xcr I0508 18:02:50.877596 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9xjjn I0508 18:02:50.877612 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-image-registry/node-ca-wb54l I0508 18:02:50.877619 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-dns/node-resolver-qshbx I0508 18:02:50.877622 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-rdmhw I0508 18:02:50.877629 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-dns/node-resolver-qshbx I0508 18:02:50.877627 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-multus/multus-zd6x6 I0508 18:02:50.877631 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-68.ec2.internal I0508 18:02:50.877638 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-multus/network-metrics-daemon-rdmhw I0508 18:02:50.877642 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-multus/multus-zd6x6 I0508 18:02:50.877600 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod kube-system/global-pull-secret-syncer-thlmm I0508 18:02:50.877707 3653 base_network_controller_pods.go:489] [default/kube-system/global-pull-secret-syncer-thlmm] creating logical port kube-system_global-pull-secret-syncer-thlmm for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:50.877713 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zzn9w I0508 18:02:50.877725 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zzn9w I0508 18:02:50.877736 3653 base_network_controller_pods.go:489] [default/openshift-multus/network-metrics-daemon-rdmhw] creating logical port openshift-multus_network-metrics-daemon-rdmhw for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:50.877627 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-image-registry/node-ca-wb54l I0508 18:02:50.877615 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9xjjn I0508 18:02:50.877802 3653 base_network_controller_pods.go:489] [default/openshift-network-diagnostics/network-check-target-9xjjn] creating logical port openshift-network-diagnostics_network-check-target-9xjjn for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:50.877598 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod kube-system/konnectivity-agent-88vrq I0508 18:02:50.877898 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod kube-system/konnectivity-agent-88vrq I0508 18:02:50.877573 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-68.ec2.internal I0508 18:02:50.877924 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-68.ec2.internal I0508 18:02:50.878556 3653 pods.go:270] [kube-system/global-pull-secret-syncer-thlmm] addLogicalPort took 860.342µs, libovsdb time 344.305µs I0508 18:02:50.878884 3653 pods.go:270] [openshift-multus/network-metrics-daemon-rdmhw] addLogicalPort took 1.155802ms, libovsdb time 569.927µs I0508 18:02:50.879019 3653 pods.go:270] [openshift-network-diagnostics/network-check-target-9xjjn] addLogicalPort took 1.225031ms, libovsdb time 624.755µs I0508 18:02:50.887379 3653 gateway_shared_intf.go:1793] Creating Gateway Node Port Watcher I0508 18:02:50.899547 3653 iptables.go:108] Creating table: mangle chain: OUTPUT I0508 18:02:50.902967 3653 iptables.go:108] Creating table: nat chain: OUTPUT I0508 18:02:50.920714 3653 iptables.go:108] Creating table: nat chain: PREROUTING I0508 18:02:51.041336 3653 iptables.go:108] Creating table: filter chain: FORWARD I0508 18:02:51.058280 3653 gateway_shared_intf.go:2208] Ensuring IP Neighbor entry for: 169.254.0.1 W0508 18:02:51.058545 3653 gateway_shared_intf.go:2214] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0508 18:02:51.058638 3653 gateway_shared_intf.go:2208] Ensuring IP Neighbor entry for: 169.254.0.4 W0508 18:02:51.058709 3653 gateway_shared_intf.go:2214] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0508 18:02:51.058755 3653 gateway_init.go:325] Gateway and management port readiness took 743.540773ms I0508 18:02:51.058783 3653 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0508 18:02:51.058818 3653 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0508 18:02:51.058829 3653 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0508 18:02:51.058838 3653 nad_controller.go:540] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0508 18:02:51.058898 3653 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0508 18:02:51.058975 3653 nad_controller.go:579] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.295µs I0508 18:02:51.059197 3653 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0508 18:02:51.059239 3653 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 33.855µs I0508 18:02:51.059249 3653 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 51.661µs I0508 18:02:51.059254 3653 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0508 18:02:51.059264 3653 nad_controller.go:378] [node-nad-controller NAD controller]: started I0508 18:02:51.059268 3653 default_node_network_controller.go:944] Starting the default node network controller I0508 18:02:51.059274 3653 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0508 18:02:51.059347 3653 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.463µs I0508 18:02:51.059448 3653 default_node_network_controller.go:1540] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0508 18:02:51.059457 3653 default_node_network_controller.go:980] Waiting for gateway and management port readiness... I0508 18:02:51.059480 3653 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc6b30 0xc0002e7030 0xc0002e70a0 0xc0002e7110 0xc0002e71f0 0xc0002e7260 0xc0002e72d0 0xc000c343c0 0xc0002e7340 0xc0002e73b0 0xc000aa2370 0xc0065728c0 0xc0002e7420 map[0x306c920:0xc002592ae0 0x306ce60:0xc002592c60 0x306db80:0xc002592a20 0x306de20:0xc002592a80 0x306e600:0xc002592b40 0x306e8a0:0xc002592ba0 0x306eb40:0xc002592c00 0x306f320:0xc002592cc0 0x306f5c0:0xc0050020c0 0x3090a40:0xc000aa7a40 0x3092440:0xc000bf15c0 0x3092e00:0xc000bb8840 0x3093140:0xc000bb87e0 0x3094180:0xc003289b00 0x3094800:0xc000978840] 0xc0003408c0 0} I0508 18:02:51.059539 3653 gateway.go:151] Starting gateway service sync I0508 18:02:51.059937 3653 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0508 18:02:51.062045 3653 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0508 18:02:51.076651 3653 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0508 18:02:51.077961 3653 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0508 18:02:51.079154 3653 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0508 18:02:51.113477 3653 gateway.go:168] Gateway service sync done. Time taken: 53.923797ms I0508 18:02:51.114513 3653 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31054 I0508 18:02:51.114634 3653 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc6b30 0xc0002e7030 0xc0002e70a0 0xc0002e7110 0xc0002e71f0 0xc0002e7260 0xc0002e72d0 0xc000c343c0 0xc0002e7340 0xc0002e73b0 0xc000aa2370 0xc0065728c0 0xc0002e7420 map[0x306c920:0xc002592ae0 0x306ce60:0xc002592c60 0x306db80:0xc002592a20 0x306de20:0xc002592a80 0x306e600:0xc002592b40 0x306e8a0:0xc002592ba0 0x306eb40:0xc002592c00 0x306f320:0xc002592cc0 0x306f5c0:0xc0050020c0 0x3090a40:0xc000aa7a40 0x3092440:0xc000bf15c0 0x3092e00:0xc000bb8840 0x3093140:0xc000bb87e0 0x3094180:0xc003289b00 0x3094800:0xc000978840] 0xc0003408c0 0} I0508 18:02:51.115030 3653 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc6b30 0xc0002e7030 0xc0002e70a0 0xc0002e7110 0xc0002e71f0 0xc0002e7260 0xc0002e72d0 0xc000c343c0 0xc0002e7340 0xc0002e73b0 0xc000aa2370 0xc0065728c0 0xc0002e7420 map[0x306c920:0xc002592ae0 0x306ce60:0xc002592c60 0x306db80:0xc002592a20 0x306de20:0xc002592a80 0x306e600:0xc002592b40 0x306e8a0:0xc002592ba0 0x306eb40:0xc002592c00 0x306f320:0xc002592cc0 0x306f5c0:0xc0050020c0 0x3090a40:0xc000aa7a40 0x3092440:0xc000bf15c0 0x3092e00:0xc000bb8840 0x3093140:0xc000bb87e0 0x3094180:0xc003289b00 0x3094800:0xc000978840] 0xc0003408c0 0} I0508 18:02:51.115064 3653 gateway.go:532] Reconciling gateway with updates I0508 18:02:51.120490 3653 gateway.go:339] Spawning Conntrack Rule Check Thread I0508 18:02:51.120571 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0508 18:02:51.120590 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0508 18:02:51.120596 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns/dns-default I0508 18:02:51.120595 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0508 18:02:51.120602 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0508 18:02:51.120603 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0508 18:02:51.120608 3653 default_node_network_controller.go:1005] Gateway and management port readiness took 61.142002ms I0508 18:02:51.120606 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0508 18:02:51.120615 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0508 18:02:51.120613 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0508 18:02:51.120620 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0508 18:02:51.120628 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0508 18:02:51.120645 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0508 18:02:51.120576 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0508 18:02:51.120654 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0508 18:02:51.120661 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0508 18:02:51.120660 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0508 18:02:51.120666 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0508 18:02:51.120676 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0508 18:02:51.120681 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0508 18:02:51.120654 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0508 18:02:51.120699 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0508 18:02:51.120705 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0508 18:02:51.120687 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-default I0508 18:02:51.120714 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0508 18:02:51.120726 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0508 18:02:51.120726 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0508 18:02:51.120735 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0508 18:02:51.120738 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0508 18:02:51.120666 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0508 18:02:51.120743 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0508 18:02:51.120745 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0508 18:02:51.120748 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0508 18:02:51.120750 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-insights/metrics I0508 18:02:51.120753 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0508 18:02:51.120755 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0508 18:02:51.120756 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0508 18:02:51.120725 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0508 18:02:51.120764 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0508 18:02:51.120768 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0508 18:02:51.120777 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0508 18:02:51.120779 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/kubernetes I0508 18:02:51.120740 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0508 18:02:51.120787 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0508 18:02:51.120789 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/kubernetes I0508 18:02:51.120788 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0508 18:02:51.120799 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0508 18:02:51.120800 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0508 18:02:51.120803 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0508 18:02:51.120761 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-insights/metrics I0508 18:02:51.120805 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0508 18:02:51.120713 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0508 18:02:51.120822 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/packageserver I0508 18:02:51.120829 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0508 18:02:51.120831 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0508 18:02:51.120835 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0508 18:02:51.120843 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0508 18:02:51.120685 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0508 18:02:51.120851 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0508 18:02:51.120854 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0508 18:02:51.120859 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0508 18:02:51.120820 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift I0508 18:02:51.120768 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0508 18:02:51.120788 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0508 18:02:51.120889 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/openshift I0508 18:02:51.120726 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0508 18:02:51.120905 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0508 18:02:51.120831 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0508 18:02:51.120663 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0508 18:02:51.120932 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0508 18:02:51.120806 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0508 18:02:51.120990 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0508 18:02:51.121007 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0508 18:02:51.120772 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0508 18:02:51.120754 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0508 18:02:51.120728 3653 port_claim.go:71] Svc openshift-ingress/router-default: port 30144 is already open, no action needed I0508 18:02:51.121084 3653 port_claim.go:71] Svc openshift-ingress/router-default: port 30378 is already open, no action needed I0508 18:02:51.120788 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0508 18:02:51.121146 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0508 18:02:51.120794 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0508 18:02:51.120807 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-apiserver I0508 18:02:51.121195 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-apiserver I0508 18:02:51.120811 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0508 18:02:51.120817 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0508 18:02:51.120812 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0508 18:02:51.120712 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0508 18:02:51.121278 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0508 18:02:51.120820 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0508 18:02:51.120833 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/packageserver I0508 18:02:51.120743 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0508 18:02:51.121398 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0508 18:02:51.121597 3653 gateway_shared_intf.go:2158] Ensuring masquerade resources on br-ex (ifindex=5, lastSuccessful=0) I0508 18:02:51.121904 3653 gateway_shared_intf.go:2017] Setting OVN Masquerade route with source: 10.0.0.68 I0508 18:02:51.122112 3653 gateway_shared_intf.go:2208] Ensuring IP Neighbor entry for: 169.254.0.1 I0508 18:02:51.122190 3653 gateway_shared_intf.go:2208] Ensuring IP Neighbor entry for: 169.254.0.4 I0508 18:02:51.122280 3653 gateway_shared_intf.go:2184] Masquerade resources ensured on br-ex (ifindex=5) I0508 18:02:51.122350 3653 node_ip_handler_linux.go:189] Node IP manager is running W0508 18:02:51.161616 3653 port_linux.go:260] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0508 18:02:51.161909 3653 port_linux.go:260] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0508 18:02:51.162100 3653 port_linux.go:276] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0508 18:02:51.242151 3653 egressfirewall.go:708] Syncing node "ip-10-0-2-114.ec2.internal" for egress firewall I0508 18:02:51.242156 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-114.ec2.internal I0508 18:02:51.242183 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-114.ec2.internal: 31.505µs I0508 18:02:51.242235 3653 default_network_controller.go:1032] Node "ip-10-0-2-114.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false + [[ 4 -lt 90 ]] + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ patch-br-ex_ip-10-0-0-68.ec2.internal-to-br-int == *to-br-int ]] + local of_port ++ ovs-vsctl get interface patch-br-ex_ip-10-0-0-68.ec2.internal-to-br-int ofport + of_port=2 + ovs-ofctl add-flow br-ex cookie=0x0305,table=0,priority=499,in_port=2,arp,arp_op=1,actions=drop ++ date -Iseconds + echo '2026-05-08T18:02:51+00:00 - INFO - Added GARP drop flow on br-ex port patch-br-ex_ip-10-0-0-68.ec2.internal-to-br-int (ofport=2)' 2026-05-08T18:02:51+00:00 - INFO - Added GARP drop flow on br-ex port patch-br-ex_ip-10-0-0-68.ec2.internal-to-br-int (ofport=2) + return 0 + return 0 I0508 18:02:52.580914 3653 default_node_network_controller.go:1460] Starting node controller node sync W0508 18:02:52.580950 3653 egressip_healthcheck.go:76] Health checking using insecure connection I0508 18:02:52.581024 3653 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0508 18:02:52.592567 3653 default_node_network_controller.go:1502] Node controller node sync done. Time taken: 11.62973ms I0508 18:02:52.592825 3653 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0508 18:02:52.593186 3653 egressservice_node.go:93] Setting up event handlers for Egress Services I0508 18:02:52.593331 3653 egressservice_node.go:183] Starting Egress Services Controller I0508 18:02:52.593349 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0508 18:02:52.593358 3653 shared_informer.go:356] "Caches are synced" controller="egressservices" I0508 18:02:52.593364 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0508 18:02:52.593369 3653 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0508 18:02:52.593373 3653 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0508 18:02:52.593378 3653 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0508 18:02:52.593383 3653 egressservice_node.go:197] Repairing Egress Services I0508 18:02:52.603388 3653 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0508 18:02:52.603400 3653 external_controller.go:278] Starting Admin Policy Based Route Controller I0508 18:02:52.603522 3653 default_node_network_controller.go:1157] Egress IP for secondary host network is disabled I0508 18:02:52.603545 3653 link_network_manager.go:119] Link manager is running I0508 18:02:52.603559 3653 default_node_network_controller.go:1178] Default node network controller initialized and ready. I0508 18:02:52.603674 3653 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0508 18:02:52.603891 3653 vrf_manager.go:128] VRF manager is running I0508 18:02:52.603969 3653 node_controller_manager.go:465] Removing flows to drop GARP I0508 18:02:52.603979 3653 gateway.go:532] Reconciling gateway with updates I0508 18:02:52.626188 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0508 18:02:52.626202 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0508 18:02:52.626208 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0508 18:02:52.626228 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0508 18:02:52.626231 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0508 18:02:52.626236 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns/dns-default I0508 18:02:52.626239 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0508 18:02:52.626240 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0508 18:02:52.626255 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0508 18:02:52.626259 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0508 18:02:52.626276 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0508 18:02:52.626292 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0508 18:02:52.626297 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0508 18:02:52.626307 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0508 18:02:52.626313 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-apiserver I0508 18:02:52.626323 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-apiserver I0508 18:02:52.626325 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0508 18:02:52.626325 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0508 18:02:52.626328 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0508 18:02:52.626335 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0508 18:02:52.626340 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift I0508 18:02:52.626348 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0508 18:02:52.626350 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0508 18:02:52.626356 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/openshift I0508 18:02:52.626355 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-insights/metrics I0508 18:02:52.626341 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0508 18:02:52.626365 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0508 18:02:52.626373 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0508 18:02:52.626375 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0508 18:02:52.626388 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/packageserver I0508 18:02:52.626394 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0508 18:02:52.626295 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0508 18:02:52.626400 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0508 18:02:52.626407 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0508 18:02:52.626410 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0508 18:02:52.626410 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0508 18:02:52.626413 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0508 18:02:52.626350 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0508 18:02:52.626369 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-insights/metrics I0508 18:02:52.626336 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0508 18:02:52.626361 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0508 18:02:52.626311 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0508 18:02:52.626363 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0508 18:02:52.626467 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0508 18:02:52.626395 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/packageserver I0508 18:02:52.626487 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0508 18:02:52.626501 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0508 18:02:52.626502 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0508 18:02:52.626489 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0508 18:02:52.626514 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0508 18:02:52.626526 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0508 18:02:52.626423 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0508 18:02:52.626530 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0508 18:02:52.626542 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/kubernetes I0508 18:02:52.626545 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0508 18:02:52.626548 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0508 18:02:52.626552 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0508 18:02:52.626563 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0508 18:02:52.626567 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0508 18:02:52.626569 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0508 18:02:52.626394 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0508 18:02:52.626581 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0508 18:02:52.626582 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0508 18:02:52.626226 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0508 18:02:52.626599 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0508 18:02:52.626385 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0508 18:02:52.626619 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0508 18:02:52.626624 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/kubernetes I0508 18:02:52.626308 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0508 18:02:52.626633 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0508 18:02:52.626473 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0508 18:02:52.626394 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0508 18:02:52.626652 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0508 18:02:52.626531 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0508 18:02:52.626658 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0508 18:02:52.626660 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0508 18:02:52.626515 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0508 18:02:52.626668 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0508 18:02:52.626539 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0508 18:02:52.626673 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0508 18:02:52.626680 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0508 18:02:52.626515 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0508 18:02:52.626682 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress/router-default I0508 18:02:52.626699 3653 port_claim.go:71] Svc openshift-ingress/router-default: port 30144 is already open, no action needed I0508 18:02:52.626707 3653 port_claim.go:71] Svc openshift-ingress/router-default: port 30378 is already open, no action needed I0508 18:02:52.626527 3653 obj_retry.go:359] nodeGateway: retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0508 18:02:52.626763 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0508 18:02:52.626663 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0508 18:02:52.626530 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0508 18:02:52.626338 3653 obj_retry.go:423] nodeGateway: adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0508 18:02:54.836517 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-68.ec2.internal I0508 18:02:54.836544 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-68.ec2.internal: 39.282µs I0508 18:02:54.866843 3653 base_network_controller_pods.go:489] [default/openshift-image-registry/image-registry-54658ddc74-lzp5m] creating logical port openshift-image-registry_image-registry-54658ddc74-lzp5m for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:54.866995 3653 kube.go:233] Patching annotations on pod openshift-image-registry/image-registry-54658ddc74-lzp5m I0508 18:02:54.887524 3653 base_network_controller_pods.go:974] [default/openshift-image-registry/image-registry-54658ddc74-lzp5m] addLogicalPort annotation time took 20.56979ms I0508 18:02:54.888708 3653 pods.go:270] [openshift-image-registry/image-registry-54658ddc74-lzp5m] addLogicalPort took 21.881216ms, libovsdb time 895.985µs I0508 18:02:54.917547 3653 base_network_controller_pods.go:489] [default/openshift-dns/dns-default-cfc5w] creating logical port openshift-dns_dns-default-cfc5w for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:54.917678 3653 kube.go:233] Patching annotations on pod openshift-dns/dns-default-cfc5w I0508 18:02:54.920453 3653 base_network_controller_pods.go:489] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29637720-trd89 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:54.920555 3653 kube.go:233] Patching annotations on pod openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 I0508 18:02:54.925028 3653 base_network_controller_pods.go:489] [default/openshift-ingress-canary/ingress-canary-zr5hg] creating logical port openshift-ingress-canary_ingress-canary-zr5hg for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:54.925080 3653 base_network_controller_pods.go:489] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-kzldz for pod on switch ip-10-0-0-68.ec2.internal I0508 18:02:54.925156 3653 kube.go:233] Patching annotations on pod openshift-ingress-canary/ingress-canary-zr5hg I0508 18:02:54.925183 3653 kube.go:233] Patching annotations on pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz I0508 18:02:54.933747 3653 base_network_controller_pods.go:974] [default/openshift-dns/dns-default-cfc5w] addLogicalPort annotation time took 16.131617ms I0508 18:02:54.934884 3653 pods.go:270] [openshift-dns/dns-default-cfc5w] addLogicalPort took 17.355892ms, libovsdb time 834.383µs I0508 18:02:54.938105 3653 base_network_controller_pods.go:974] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89] addLogicalPort annotation time took 17.569407ms I0508 18:02:54.939286 3653 pods.go:270] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89] addLogicalPort took 18.846941ms, libovsdb time 829.407µs I0508 18:02:54.941371 3653 base_network_controller_pods.go:974] [default/openshift-ingress-canary/ingress-canary-zr5hg] addLogicalPort annotation time took 16.250115ms I0508 18:02:54.942165 3653 base_network_controller_pods.go:974] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz] addLogicalPort annotation time took 17.012882ms I0508 18:02:54.942431 3653 pods.go:270] [openshift-ingress-canary/ingress-canary-zr5hg] addLogicalPort took 17.453869ms, libovsdb time 844.687µs I0508 18:02:54.943615 3653 pods.go:270] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz] addLogicalPort took 18.544432ms, libovsdb time 978.829µs I0508 18:02:55.338040 3653 cni.go:454] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key ] I0508 18:02:55.352346 3653 cni.go:454] [openshift-ingress-canary/ingress-canary-zr5hg 6de53cb2bd33b8abb0a1faa1168c5643b18c31b0ff6c033b0e3903a43670bc89 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-zr5hg 6de53cb2bd33b8abb0a1faa1168c5643b18c31b0ff6c033b0e3903a43670bc89 network default NAD default NAD key ] I0508 18:02:55.368497 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-zr5hg, hostIfaceName: 6de53cb2bd33b8a, network: default, NAD default, SandboxID: "6de53cb2bd33b8abb0a1faa1168c5643b18c31b0ff6c033b0e3903a43670bc89", PCI device ID: , UID: "e6fcb517-a2b9-4ed0-9d37-12157f83a8f0", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0508 18:02:55.368755 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29637720-trd89, hostIfaceName: 17e3916cf33417d, network: default, NAD default, SandboxID: "17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e", PCI device ID: , UID: "fe6056f6-a77f-4eb3-ab8b-7bccbc585a28", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0508 18:02:55.378114 3653 cni.go:454] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz 3b62d0f2001abfb01bb99826287bcf2d61211bd82dc9fb6e5a0715cf15371665 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz 3b62d0f2001abfb01bb99826287bcf2d61211bd82dc9fb6e5a0715cf15371665 network default NAD default NAD key ] I0508 18:02:55.390645 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-kzldz, hostIfaceName: 3b62d0f2001abfb, network: default, NAD default, SandboxID: "3b62d0f2001abfb01bb99826287bcf2d61211bd82dc9fb6e5a0715cf15371665", PCI device ID: , UID: "d771ae6d-680a-40e5-bc96-577c32328f27", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0508 18:02:55.422029 3653 cni.go:480] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:76:3b:74:79:49\",\"name\":\"17e3916cf33417d\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2435412d-b98d-4710-b416-9b63730bf4ec\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:02:55.428849 3653 cni.go:480] [openshift-ingress-canary/ingress-canary-zr5hg 6de53cb2bd33b8abb0a1faa1168c5643b18c31b0ff6c033b0e3903a43670bc89 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-zr5hg 6de53cb2bd33b8abb0a1faa1168c5643b18c31b0ff6c033b0e3903a43670bc89 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:e2:74:87:49:22\",\"name\":\"6de53cb2bd33b8a\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5100347a-ea15-4aa8-9c53-7019fa654c05\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:02:55.440484 3653 cni.go:480] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz 3b62d0f2001abfb01bb99826287bcf2d61211bd82dc9fb6e5a0715cf15371665 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-kzldz 3b62d0f2001abfb01bb99826287bcf2d61211bd82dc9fb6e5a0715cf15371665 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:32:dd:27:72:7b\",\"name\":\"3b62d0f2001abfb\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e8ec273-fc3c-4e46-be37-643118e4939f\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:02:56.316657 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-114.ec2.internal in network "default" I0508 18:02:56.316704 3653 node_tracker.go:235] Node ip-10-0-2-114.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-114.ec2.internal" I0508 18:02:56.316702 3653 default_network_controller.go:1032] Node "ip-10-0-2-114.ec2.internal" in remote zone "ip-10-0-2-114.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:02:56.316719 3653 node_tracker.go:169] Node ip-10-0-2-114.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:56.316737 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:56.394180 3653 default_network_controller.go:1032] Node "ip-10-0-2-114.ec2.internal" in remote zone "ip-10-0-2-114.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:02:56.473184 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-114.ec2.internal in network "default" I0508 18:02:56.473256 3653 node_tracker.go:169] Node ip-10-0-2-114.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:56.473253 3653 default_network_controller.go:1032] Node "ip-10-0-2-114.ec2.internal" in remote zone "ip-10-0-2-114.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:02:56.473270 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:56.474258 3653 zone_ic_handler.go:195] Time taken to create transit switch: 328.805µs I0508 18:02:56.474273 3653 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-114.ec2.internal for the network default I0508 18:02:56.475430 3653 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-114.ec2.internal" on network "default" took: 1.512249ms I0508 18:02:59.035562 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-114.ec2.internal in network "default" I0508 18:02:59.035615 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-114.ec2.internal I0508 18:02:59.035616 3653 egressfirewall.go:708] Syncing node "ip-10-0-2-114.ec2.internal" for egress firewall I0508 18:02:59.035631 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-114.ec2.internal: 20.92µs I0508 18:02:59.035634 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-2-114.ec2.internal", IP: 10.0.2.114 to PMTUD blocking rules I0508 18:02:59.035647 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-2-114.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0508 18:02:59.035638 3653 node_tracker.go:169] Node ip-10-0-2-114.ec2.internal switch + router changed, syncing services in network "default" I0508 18:02:59.035682 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:02:59.042627 3653 default_network_controller.go:1091] Egress IP detected IP address change for node ip-10-0-2-114.ec2.internal. Updating no re-route policies I0508 18:02:59.050497 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-2-114.ec2.internal", IP: 10.0.2.114 to PMTUD blocking rules I0508 18:02:59.050516 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-2-114.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0508 18:03:02.518082 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-114.ec2.internal I0508 18:03:02.518119 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-114.ec2.internal: 46.439µs W0508 18:03:03.213762 3653 util.go:863] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0508 18:03:10.440552 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0508 18:03:12.396759 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-165.ec2.internal I0508 18:03:12.396781 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-165.ec2.internal: 29.572µs I0508 18:03:12.396799 3653 egressfirewall.go:708] Syncing node "ip-10-0-1-165.ec2.internal" for egress firewall I0508 18:03:12.396828 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-165.ec2.internal in network "default" I0508 18:03:12.396845 3653 node_tracker.go:220] Node ip-10-0-1-165.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0508 18:03:12.396865 3653 default_network_controller.go:839] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0508 18:03:12.396882 3653 obj_retry.go:605] default/networkController: failed to create *v1.Node ip-10-0-1-165.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-165.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-165.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-165.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-165.ec2.internal, error parsing annotation for node ip-10-0-1-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0508 18:03:12.410856 3653 subnet_annotations.go:148] Failed to unmarshal old node ip-10-0-1-165.ec2.internal annotation: unexpected end of JSON input I0508 18:03:12.410882 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true E0508 18:03:12.410911 3653 default_network_controller.go:1049] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0508 18:03:12.430398 3653 subnet_annotations.go:148] Failed to unmarshal old node ip-10-0-1-165.ec2.internal annotation: unexpected end of JSON input I0508 18:03:12.430441 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0508 18:03:12.430471 3653 default_network_controller.go:1049] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0508 18:03:12.444455 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-165.ec2.internal I0508 18:03:12.444471 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-165.ec2.internal: 18.531µs E0508 18:03:12.444510 3653 subnet_annotations.go:148] Failed to unmarshal old node ip-10-0-1-165.ec2.internal annotation: unexpected end of JSON input I0508 18:03:12.444515 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:12.444530 3653 egressfirewall.go:708] Syncing node "ip-10-0-1-165.ec2.internal" for egress firewall E0508 18:03:12.444564 3653 default_network_controller.go:1049] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0508 18:03:12.513100 3653 subnet_annotations.go:148] Failed to unmarshal old node ip-10-0-1-165.ec2.internal annotation: unexpected end of JSON input E0508 18:03:12.513114 3653 subnet_annotations.go:148] Failed to unmarshal old node ip-10-0-1-165.ec2.internal annotation: unexpected end of JSON input I0508 18:03:12.513134 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:12.638796 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-165.ec2.internal I0508 18:03:12.638813 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-165.ec2.internal: 20.977µs I0508 18:03:12.638858 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:12.638866 3653 egressfirewall.go:708] Syncing node "ip-10-0-1-165.ec2.internal" for egress firewall I0508 18:03:12.653289 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:12.832337 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:15.717411 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0508 18:03:16.240267 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0508 18:03:20.243651 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Node ip-10-0-1-165.ec2.internal I0508 18:03:20.243676 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Node ip-10-0-1-165.ec2.internal I0508 18:03:20.243765 3653 obj_retry.go:434] default/networkController: retry add failed for *v1.Node ip-10-0-1-165.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-165.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-165.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-165.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-165.ec2.internal I0508 18:03:20.879329 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-qsjfg I0508 18:03:20.879351 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-jz27t I0508 18:03:20.879364 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-multus/network-metrics-daemon-qsjfg I0508 18:03:20.879373 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod kube-system/global-pull-secret-syncer-jz27t I0508 18:03:20.879408 3653 obj_retry.go:434] default/networkController: retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-qsjfg, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-qsjfg: suppressed error logged: pod openshift-multus/network-metrics-daemon-qsjfg: no pod IPs found I0508 18:03:20.879331 3653 obj_retry.go:359] default/networkController: retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-2mfkk I0508 18:03:20.879411 3653 obj_retry.go:434] default/networkController: retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-jz27t, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-jz27t: suppressed error logged: pod kube-system/global-pull-secret-syncer-jz27t: no pod IPs found I0508 18:03:20.879422 3653 obj_retry.go:423] default/networkController: adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-2mfkk I0508 18:03:20.879450 3653 obj_retry.go:434] default/networkController: retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-2mfkk, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-2mfkk: suppressed error logged: pod openshift-network-diagnostics/network-check-target-2mfkk: no pod IPs found I0508 18:03:22.680460 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:27.907161 3653 cni.go:454] [kube-system/global-pull-secret-syncer-thlmm c0745d753e4ff0331d5e85472f5bdcd172b28e8a585e711554311ad85fb48bb1 network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-thlmm c0745d753e4ff0331d5e85472f5bdcd172b28e8a585e711554311ad85fb48bb1 network default NAD default NAD key ] I0508 18:03:27.917144 3653 helper_linux.go:546] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-thlmm, hostIfaceName: c0745d753e4ff03, network: default, NAD default, SandboxID: "c0745d753e4ff0331d5e85472f5bdcd172b28e8a585e711554311ad85fb48bb1", PCI device ID: , UID: "3d3134ad-8f41-4a9a-b480-fa8ba6b0b72e", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0508 18:03:27.962856 3653 cni.go:480] [kube-system/global-pull-secret-syncer-thlmm c0745d753e4ff0331d5e85472f5bdcd172b28e8a585e711554311ad85fb48bb1 network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-thlmm c0745d753e4ff0331d5e85472f5bdcd172b28e8a585e711554311ad85fb48bb1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:3d:cf:28:6c:dc\",\"name\":\"c0745d753e4ff03\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6051742c-8827-4d6b-97e0-1a1855c17bdb\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:03:28.202406 3653 cni.go:454] [openshift-network-diagnostics/network-check-target-9xjjn 3fb9d6c79335e7c5a2ea8adf67a703441ec5645e7d6268a15b789699f0032776 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-9xjjn 3fb9d6c79335e7c5a2ea8adf67a703441ec5645e7d6268a15b789699f0032776 network default NAD default NAD key ] I0508 18:03:28.213237 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-9xjjn, hostIfaceName: 3fb9d6c79335e7c, network: default, NAD default, SandboxID: "3fb9d6c79335e7c5a2ea8adf67a703441ec5645e7d6268a15b789699f0032776", PCI device ID: , UID: "546b7fc9-d1d7-4e71-9d32-1ca23b7608cb", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0508 18:03:28.258913 3653 cni.go:480] [openshift-network-diagnostics/network-check-target-9xjjn 3fb9d6c79335e7c5a2ea8adf67a703441ec5645e7d6268a15b789699f0032776 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-9xjjn 3fb9d6c79335e7c5a2ea8adf67a703441ec5645e7d6268a15b789699f0032776 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:2e:74:75:0b:88\",\"name\":\"3fb9d6c79335e7c\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b599c96b-b33b-49ed-ae87-986e46d94ebb\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:03:30.319334 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:30.319373 3653 egressfirewall.go:708] Syncing node "ip-10-0-1-165.ec2.internal" for egress firewall I0508 18:03:30.319347 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-165.ec2.internal I0508 18:03:30.319413 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-165.ec2.internal: 73.454µs I0508 18:03:32.912091 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0508 18:03:33.303342 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0508 18:03:35.482172 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-165.ec2.internal in network "default" I0508 18:03:35.482200 3653 node_tracker.go:235] Node ip-10-0-1-165.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-165.ec2.internal" I0508 18:03:35.482208 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "ip-10-0-1-165.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:35.482232 3653 node_tracker.go:169] Node ip-10-0-1-165.ec2.internal switch + router changed, syncing services in network "default" I0508 18:03:35.482290 3653 services_controller.go:574] Full service sync requested for network=default W0508 18:03:35.482387 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0508 18:03:35.626592 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-165.ec2.internal in network "default" I0508 18:03:35.626633 3653 default_network_controller.go:1032] Node "ip-10-0-1-165.ec2.internal" in remote zone "ip-10-0-1-165.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0508 18:03:35.626647 3653 node_tracker.go:169] Node ip-10-0-1-165.ec2.internal switch + router changed, syncing services in network "default" I0508 18:03:35.626660 3653 services_controller.go:574] Full service sync requested for network=default W0508 18:03:35.627022 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0508 18:03:35.627632 3653 zone_ic_handler.go:195] Time taken to create transit switch: 281.164µs I0508 18:03:35.627653 3653 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-165.ec2.internal for the network default I0508 18:03:35.628693 3653 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-165.ec2.internal" on network "default" took: 1.353252ms I0508 18:03:37.687029 3653 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-165.ec2.internal in network "default" I0508 18:03:37.687074 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-1-165.ec2.internal", IP: 10.0.1.165 to PMTUD blocking rules I0508 18:03:37.687087 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-1-165.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0508 18:03:37.687094 3653 node_tracker.go:169] Node ip-10-0-1-165.ec2.internal switch + router changed, syncing services in network "default" I0508 18:03:37.687110 3653 services_controller.go:574] Full service sync requested for network=default I0508 18:03:37.687123 3653 egressfirewall.go:708] Syncing node "ip-10-0-1-165.ec2.internal" for egress firewall I0508 18:03:37.687136 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-165.ec2.internal I0508 18:03:37.687148 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-165.ec2.internal: 15.155µs W0508 18:03:37.687331 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0508 18:03:37.693257 3653 default_network_controller.go:1091] Egress IP detected IP address change for node ip-10-0-1-165.ec2.internal. Updating no re-route policies I0508 18:03:37.704123 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-1-165.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0508 18:03:37.704139 3653 default_node_network_controller.go:1373] Adding remote node "ip-10-0-1-165.ec2.internal", IP: 10.0.1.165 to PMTUD blocking rules I0508 18:03:41.199672 3653 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-165.ec2.internal I0508 18:03:41.199709 3653 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-165.ec2.internal: 45.084µs W0508 18:03:41.454052 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0508 18:03:41.454077 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0508 18:04:04.290309 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0508 18:04:33.819154 3653 base_network_controller_pods.go:489] [default/openshift-network-diagnostics/network-check-source-9f64469cd-ftpct] creating logical port openshift-network-diagnostics_network-check-source-9f64469cd-ftpct for pod on switch ip-10-0-0-68.ec2.internal I0508 18:04:33.819292 3653 kube.go:233] Patching annotations on pod openshift-network-diagnostics/network-check-source-9f64469cd-ftpct I0508 18:04:33.837557 3653 base_network_controller_pods.go:974] [default/openshift-network-diagnostics/network-check-source-9f64469cd-ftpct] addLogicalPort annotation time took 18.339352ms I0508 18:04:33.838604 3653 pods.go:270] [openshift-network-diagnostics/network-check-source-9f64469cd-ftpct] addLogicalPort took 19.466631ms, libovsdb time 785.753µs I0508 18:04:33.960419 3653 base_network_controller_pods.go:489] [default/openshift-insights/insights-operator-b6bf49759-k7fs4] creating logical port openshift-insights_insights-operator-b6bf49759-k7fs4 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:04:33.960535 3653 kube.go:233] Patching annotations on pod openshift-insights/insights-operator-b6bf49759-k7fs4 I0508 18:04:33.980421 3653 base_network_controller_pods.go:974] [default/openshift-insights/insights-operator-b6bf49759-k7fs4] addLogicalPort annotation time took 19.93357ms I0508 18:04:33.981639 3653 pods.go:270] [openshift-insights/insights-operator-b6bf49759-k7fs4] addLogicalPort took 21.269535ms, libovsdb time 933.156µs I0508 18:04:34.172613 3653 cni.go:454] [openshift-network-diagnostics/network-check-source-9f64469cd-ftpct 36d4483b20a3fe93414c94eee257368918321d6c9dbfd45d69db879977ee6969 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-9f64469cd-ftpct 36d4483b20a3fe93414c94eee257368918321d6c9dbfd45d69db879977ee6969 network default NAD default NAD key ] I0508 18:04:34.183005 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-9f64469cd-ftpct, hostIfaceName: 36d4483b20a3fe9, network: default, NAD default, SandboxID: "36d4483b20a3fe93414c94eee257368918321d6c9dbfd45d69db879977ee6969", PCI device ID: , UID: "3f917c63-9acc-4af9-a4d6-afbbf31b2179", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0508 18:04:34.229251 3653 cni.go:480] [openshift-network-diagnostics/network-check-source-9f64469cd-ftpct 36d4483b20a3fe93414c94eee257368918321d6c9dbfd45d69db879977ee6969 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-9f64469cd-ftpct 36d4483b20a3fe93414c94eee257368918321d6c9dbfd45d69db879977ee6969 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:73:2c:21:56:6b\",\"name\":\"36d4483b20a3fe9\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b378e34e-6ee7-41d1-86dc-1baa3982d4e4\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:04:34.315161 3653 cni.go:454] [openshift-insights/insights-operator-b6bf49759-k7fs4 489d1a166b9dc0e19da7471af43dab3b5035415088fe518aebd849f3e83b7906 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-b6bf49759-k7fs4 489d1a166b9dc0e19da7471af43dab3b5035415088fe518aebd849f3e83b7906 network default NAD default NAD key ] I0508 18:04:34.326121 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-b6bf49759-k7fs4, hostIfaceName: 489d1a166b9dc0e, network: default, NAD default, SandboxID: "489d1a166b9dc0e19da7471af43dab3b5035415088fe518aebd849f3e83b7906", PCI device ID: , UID: "8c0b054f-2d7e-44c0-a6c6-db1b68ac6378", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0508 18:04:34.380568 3653 cni.go:480] [openshift-insights/insights-operator-b6bf49759-k7fs4 489d1a166b9dc0e19da7471af43dab3b5035415088fe518aebd849f3e83b7906 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-b6bf49759-k7fs4 489d1a166b9dc0e19da7471af43dab3b5035415088fe518aebd849f3e83b7906 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:cb:62:cf:a0:57\",\"name\":\"489d1a166b9dc0e\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46024a62-e5c3-44c4-bf6a-7804cb2ab7b9\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:04:35.669866 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-cluster-storage-operator/allow-egress-to-api-server to delete 1 local pods... I0508 18:04:35.669899 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-cluster-storage-operator/allow-to-dns to delete 1 local pods... I0508 18:04:35.669908 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-cluster-storage-operator/default-deny to delete 1 local pods... W0508 18:04:35.669951 3653 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-cluster-storage-operator/volume-data-source-validator-5c64b584df-qtkrx IP address from the namespace address-set, err: pod openshift-cluster-storage-operator/volume-data-source-validator-5c64b584df-qtkrx: no pod IPs found I0508 18:04:37.148049 3653 namespace.go:97] [openshift-kube-storage-version-migrator] adding namespace I0508 18:04:37.148908 3653 namespace.go:101] [openshift-kube-storage-version-migrator] adding namespace took 835.471µs I0508 18:04:37.186988 3653 namespace.go:141] [openshift-kube-storage-version-migrator] updating namespace I0508 18:04:37.296936 3653 base_network_controller_pods.go:489] [default/openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj] creating logical port openshift-kube-storage-version-migrator_migrator-9c86b5f5-nzbgj for pod on switch ip-10-0-0-68.ec2.internal I0508 18:04:37.297078 3653 kube.go:233] Patching annotations on pod openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj I0508 18:04:37.314800 3653 base_network_controller_pods.go:974] [default/openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj] addLogicalPort annotation time took 17.755893ms I0508 18:04:37.316179 3653 pods.go:270] [openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj] addLogicalPort took 19.262638ms, libovsdb time 964.423µs I0508 18:04:37.650270 3653 cni.go:454] [openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj 5b7189760d55aacb30c65ff504ed7b69a747a215032eb424d629e14b2a55733b network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj 5b7189760d55aacb30c65ff504ed7b69a747a215032eb424d629e14b2a55733b network default NAD default NAD key ] I0508 18:04:37.660394 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-9c86b5f5-nzbgj, hostIfaceName: 5b7189760d55aac, network: default, NAD default, SandboxID: "5b7189760d55aacb30c65ff504ed7b69a747a215032eb424d629e14b2a55733b", PCI device ID: , UID: "6112f503-30f2-4d57-b11c-dff8faebfeed", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0508 18:04:37.706051 3653 cni.go:480] [openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj 5b7189760d55aacb30c65ff504ed7b69a747a215032eb424d629e14b2a55733b network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-9c86b5f5-nzbgj 5b7189760d55aacb30c65ff504ed7b69a747a215032eb424d629e14b2a55733b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:fe:88:86:d0:2a\",\"name\":\"5b7189760d55aac\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f48b76e-61d5-4cbf-a458-797b3799de06\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:04:37.856787 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0508 18:04:37.856812 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0508 18:04:38.666352 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0508 18:04:40.753431 3653 namespace.go:97] [openshift-service-ca] adding namespace I0508 18:04:40.754380 3653 namespace.go:101] [openshift-service-ca] adding namespace took 930.462µs I0508 18:04:40.771023 3653 namespace.go:141] [openshift-service-ca] updating namespace I0508 18:04:42.716724 3653 namespace.go:141] [openshift-deployment-validation-operator] updating namespace I0508 18:04:42.727107 3653 namespace.go:141] [openshift-deployment-validation-operator] updating namespace W0508 18:04:50.094819 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0508 18:05:02.196166 3653 base_network_controller_pods.go:489] [default/openshift-insights/insights-runtime-extractor-d4dpj] creating logical port openshift-insights_insights-runtime-extractor-d4dpj for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:02.196290 3653 kube.go:233] Patching annotations on pod openshift-insights/insights-runtime-extractor-d4dpj I0508 18:05:02.221887 3653 base_network_controller_pods.go:974] [default/openshift-insights/insights-runtime-extractor-d4dpj] addLogicalPort annotation time took 25.637165ms I0508 18:05:02.223093 3653 pods.go:270] [openshift-insights/insights-runtime-extractor-d4dpj] addLogicalPort took 26.944136ms, libovsdb time 827.918µs I0508 18:05:02.525959 3653 pods.go:175] Deleting pod: openshift-image-registry/image-registry-54658ddc74-lzp5m I0508 18:05:02.527870 3653 pods.go:235] Attempting to release IPs for pod: openshift-image-registry/image-registry-54658ddc74-lzp5m, ips: 10.128.0.6 I0508 18:05:02.545640 3653 base_network_controller_pods.go:489] [default/openshift-image-registry/image-registry-7b8c985dd6-j69gv] creating logical port openshift-image-registry_image-registry-7b8c985dd6-j69gv for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:02.545790 3653 kube.go:233] Patching annotations on pod openshift-image-registry/image-registry-7b8c985dd6-j69gv I0508 18:05:02.554336 3653 cni.go:454] [openshift-insights/insights-runtime-extractor-d4dpj 40ab0b3006a39d036ceb520ca1a018dfef9b62a5622873cf58b590c19de69162 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-d4dpj 40ab0b3006a39d036ceb520ca1a018dfef9b62a5622873cf58b590c19de69162 network default NAD default NAD key ] I0508 18:05:02.556652 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-54658ddc74-lzp5m I0508 18:05:02.556686 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-54658ddc74-lzp5m I0508 18:05:02.563913 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-d4dpj, hostIfaceName: 40ab0b3006a39d0, network: default, NAD default, SandboxID: "40ab0b3006a39d036ceb520ca1a018dfef9b62a5622873cf58b590c19de69162", PCI device ID: , UID: "9b481250-3db2-4799-a216-ee732cba1aa5", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0508 18:05:02.567977 3653 base_network_controller_pods.go:974] [default/openshift-image-registry/image-registry-7b8c985dd6-j69gv] addLogicalPort annotation time took 22.225748ms I0508 18:05:02.569457 3653 pods.go:270] [openshift-image-registry/image-registry-7b8c985dd6-j69gv] addLogicalPort took 23.833495ms, libovsdb time 1.025964ms I0508 18:05:02.613951 3653 cni.go:480] [openshift-insights/insights-runtime-extractor-d4dpj 40ab0b3006a39d036ceb520ca1a018dfef9b62a5622873cf58b590c19de69162 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-d4dpj 40ab0b3006a39d036ceb520ca1a018dfef9b62a5622873cf58b590c19de69162 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:48:f1:f7:80:7c\",\"name\":\"40ab0b3006a39d0\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0793b796-16f5-4a96-9437-15e6c40108bd\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:05:02.902796 3653 cni.go:454] [openshift-image-registry/image-registry-7b8c985dd6-j69gv d02449f7ff81a3b448a03d71f5a57deaf79e2796d77272c9746ffd62d2df69a8 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7b8c985dd6-j69gv d02449f7ff81a3b448a03d71f5a57deaf79e2796d77272c9746ffd62d2df69a8 network default NAD default NAD key ] I0508 18:05:02.912937 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7b8c985dd6-j69gv, hostIfaceName: d02449f7ff81a3b, network: default, NAD default, SandboxID: "d02449f7ff81a3b448a03d71f5a57deaf79e2796d77272c9746ffd62d2df69a8", PCI device ID: , UID: "2d23aaa4-893c-4dad-a06b-57eb12eb172a", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0508 18:05:02.959284 3653 cni.go:480] [openshift-image-registry/image-registry-7b8c985dd6-j69gv d02449f7ff81a3b448a03d71f5a57deaf79e2796d77272c9746ffd62d2df69a8 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7b8c985dd6-j69gv d02449f7ff81a3b448a03d71f5a57deaf79e2796d77272c9746ffd62d2df69a8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:d0:77:58:1d:f9\",\"name\":\"d02449f7ff81a3b\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ff931fa-62cc-4cc3-8f40-529919d9a0c7\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:05:03.046613 3653 cni.go:454] [openshift-dns/dns-default-cfc5w ea2f5ce33a77ef3d6b3f14de9206bcf2bb928d2cf69de96e830743e767ac2dc5 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-cfc5w ea2f5ce33a77ef3d6b3f14de9206bcf2bb928d2cf69de96e830743e767ac2dc5 network default NAD default NAD key ] I0508 18:05:03.056154 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-dns, podName: dns-default-cfc5w, hostIfaceName: ea2f5ce33a77ef3, network: default, NAD default, SandboxID: "ea2f5ce33a77ef3d6b3f14de9206bcf2bb928d2cf69de96e830743e767ac2dc5", PCI device ID: , UID: "109e7a5a-756c-49f2-8284-54798b90fcb9", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0508 18:05:03.105152 3653 cni.go:480] [openshift-dns/dns-default-cfc5w ea2f5ce33a77ef3d6b3f14de9206bcf2bb928d2cf69de96e830743e767ac2dc5 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-cfc5w ea2f5ce33a77ef3d6b3f14de9206bcf2bb928d2cf69de96e830743e767ac2dc5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:35:27:f8:0a:35\",\"name\":\"ea2f5ce33a77ef3\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/543ca3b1-380b-434a-9fde-6dcc88ee69a9\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:05:03.540174 3653 util.go:863] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0508 18:05:03.735635 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-59b88794fc-tw8s2 I0508 18:05:03.735664 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-59b88794fc-tw8s2 W0508 18:05:04.723490 3653 util.go:863] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0508 18:05:05.539771 3653 util.go:863] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:05.539792 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:05.539798 3653 util.go:863] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:06.738396 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0508 18:05:06.738416 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.738431 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.738439 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.738422 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0508 18:05:06.744691 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0508 18:05:06.744703 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0508 18:05:06.751027 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.751043 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.751048 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.762456 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.762473 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.762479 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0508 18:05:06.762561 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0508 18:05:06.762581 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0508 18:05:08.848773 3653 base_network_controller_pods.go:489] [default/openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-7f68c558c5-gw2wj for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:08.848870 3653 kube.go:233] Patching annotations on pod openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj I0508 18:05:08.863768 3653 base_network_controller_pods.go:974] [default/openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj] addLogicalPort annotation time took 14.93281ms I0508 18:05:08.864982 3653 pods.go:270] [openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj] addLogicalPort took 16.222269ms, libovsdb time 852.203µs I0508 18:05:09.803295 3653 cni.go:454] [openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj 174bd8ccc305a05f528f6ecbc61f2c0fddba5f8bfea5a775e68c50be31730e34 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj 174bd8ccc305a05f528f6ecbc61f2c0fddba5f8bfea5a775e68c50be31730e34 network default NAD default NAD key ] I0508 18:05:09.812514 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-7f68c558c5-gw2wj, hostIfaceName: 174bd8ccc305a05, network: default, NAD default, SandboxID: "174bd8ccc305a05f528f6ecbc61f2c0fddba5f8bfea5a775e68c50be31730e34", PCI device ID: , UID: "2fff15b3-af1b-4277-81dd-94ee07ae6b07", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0508 18:05:09.859537 3653 cni.go:480] [openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj 174bd8ccc305a05f528f6ecbc61f2c0fddba5f8bfea5a775e68c50be31730e34 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7f68c558c5-gw2wj 174bd8ccc305a05f528f6ecbc61f2c0fddba5f8bfea5a775e68c50be31730e34 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:16:3b:3b:08:14\",\"name\":\"174bd8ccc305a05\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/248270df-7a7e-421f-be81-183c746d23f8\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:05:11.558148 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0508 18:05:11.579317 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0508 18:05:11.766786 3653 util.go:863] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0508 18:05:11.936366 3653 base_network_controller_pods.go:489] [default/openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7] creating logical port openshift-monitoring_prometheus-operator-78cff5dd4d-v7vh7 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:11.936495 3653 kube.go:233] Patching annotations on pod openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7 I0508 18:05:11.953098 3653 base_network_controller_pods.go:974] [default/openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7] addLogicalPort annotation time took 16.650966ms I0508 18:05:11.954231 3653 pods.go:270] [openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7] addLogicalPort took 17.871358ms, libovsdb time 767.053µs I0508 18:05:12.289480 3653 cni.go:454] [openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7 34b1e417df0402cccdbba03f98b0b46d380059534ce621efc753738c43fcf60d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7 34b1e417df0402cccdbba03f98b0b46d380059534ce621efc753738c43fcf60d network default NAD default NAD key ] I0508 18:05:12.299824 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-78cff5dd4d-v7vh7, hostIfaceName: 34b1e417df0402c, network: default, NAD default, SandboxID: "34b1e417df0402cccdbba03f98b0b46d380059534ce621efc753738c43fcf60d", PCI device ID: , UID: "17523ef5-43e0-4dbf-a03b-1f1b2a386f3b", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0508 18:05:12.346918 3653 cni.go:480] [openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7 34b1e417df0402cccdbba03f98b0b46d380059534ce621efc753738c43fcf60d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-78cff5dd4d-v7vh7 34b1e417df0402cccdbba03f98b0b46d380059534ce621efc753738c43fcf60d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:d5:75:c1:6d:3e\",\"name\":\"34b1e417df0402c\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6bbf4399-799a-4dce-844f-67ea00039097\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:05:12.770182 3653 util.go:863] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:12.770205 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:12.770226 3653 util.go:863] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:15.544273 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:15.544299 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0508 18:05:15.544306 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0508 18:05:17.586928 3653 base_network_controller_pods.go:489] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:17.587038 3653 kube.go:233] Patching annotations on pod openshift-monitoring/alertmanager-main-0 I0508 18:05:17.614394 3653 base_network_controller_pods.go:974] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 27.390112ms I0508 18:05:17.615973 3653 pods.go:270] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 29.060831ms, libovsdb time 1.207547ms I0508 18:05:17.940695 3653 cni.go:454] [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key ] I0508 18:05:17.950788 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 3e632b88dcb4c64, network: default, NAD default, SandboxID: "3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4", PCI device ID: , UID: "08a9c6f6-8ea2-4258-83fd-8d7384f5d8b0", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0508 18:05:17.997510 3653 cni.go:480] [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:be:eb:53:6a:e1\",\"name\":\"3e632b88dcb4c64\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f863490-f0e3-4249-afeb-aa8ffaacbda4\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:05:19.613500 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:19.613524 3653 util.go:863] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:19.613534 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:19.850514 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:19.850540 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:19.850549 3653 util.go:863] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0508 18:05:21.110738 3653 base_network_controller_pods.go:489] [default/openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c] creating logical port openshift-monitoring_monitoring-plugin-5b65996cf7-zpb6c for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:21.110843 3653 kube.go:233] Patching annotations on pod openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c I0508 18:05:21.124440 3653 base_network_controller_pods.go:974] [default/openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c] addLogicalPort annotation time took 13.634121ms I0508 18:05:21.125889 3653 pods.go:270] [openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c] addLogicalPort took 15.170045ms, libovsdb time 944.849µs I0508 18:05:22.089287 3653 cni.go:454] [openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c fb2531409e7d4f86d9cafdfafd06556ae7d9ae90fcc42163ef6c7a3c1a2e3839 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c fb2531409e7d4f86d9cafdfafd06556ae7d9ae90fcc42163ef6c7a3c1a2e3839 network default NAD default NAD key ] I0508 18:05:22.098398 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5b65996cf7-zpb6c, hostIfaceName: fb2531409e7d4f8, network: default, NAD default, SandboxID: "fb2531409e7d4f86d9cafdfafd06556ae7d9ae90fcc42163ef6c7a3c1a2e3839", PCI device ID: , UID: "3a05d1a2-ef41-4f61-a2f5-b0869b746cb0", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0508 18:05:22.147056 3653 cni.go:480] [openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c fb2531409e7d4f86d9cafdfafd06556ae7d9ae90fcc42163ef6c7a3c1a2e3839 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5b65996cf7-zpb6c fb2531409e7d4f86d9cafdfafd06556ae7d9ae90fcc42163ef6c7a3c1a2e3839 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:e3:1a:ef:71:ec\",\"name\":\"fb2531409e7d4f8\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3442b591-b967-41d5-9346-e89c0c408c38\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:05:23.619656 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0508 18:05:23.636745 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0508 18:05:23.813760 3653 util.go:863] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.813780 3653 util.go:863] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.813786 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.813791 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.833697 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0508 18:05:23.869323 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.869343 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.869351 3653 util.go:863] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.869359 3653 util.go:863] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0508 18:05:23.890155 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0508 18:05:23.964109 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0508 18:05:23.964133 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0508 18:05:24.538968 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0508 18:05:24.833363 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0508 18:05:24.833384 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0508 18:05:32.081746 3653 base_network_controller_pods.go:489] [default/openshift-console/downloads-84f7f6d49-q2mf8] creating logical port openshift-console_downloads-84f7f6d49-q2mf8 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:32.081858 3653 kube.go:233] Patching annotations on pod openshift-console/downloads-84f7f6d49-q2mf8 I0508 18:05:32.101660 3653 base_network_controller_pods.go:974] [default/openshift-console/downloads-84f7f6d49-q2mf8] addLogicalPort annotation time took 19.836179ms I0508 18:05:32.102981 3653 pods.go:270] [openshift-console/downloads-84f7f6d49-q2mf8] addLogicalPort took 21.256463ms, libovsdb time 983.339µs I0508 18:05:32.437885 3653 cni.go:454] [openshift-console/downloads-84f7f6d49-q2mf8 ddacbcd5c28a4892bfb8b4e51a089c4c627e050c12198dfb80c16357d0a99ce4 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-84f7f6d49-q2mf8 ddacbcd5c28a4892bfb8b4e51a089c4c627e050c12198dfb80c16357d0a99ce4 network default NAD default NAD key ] I0508 18:05:32.447104 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-console, podName: downloads-84f7f6d49-q2mf8, hostIfaceName: ddacbcd5c28a489, network: default, NAD default, SandboxID: "ddacbcd5c28a4892bfb8b4e51a089c4c627e050c12198dfb80c16357d0a99ce4", PCI device ID: , UID: "2d1e9ac5-e80c-4528-b4ec-98d6a8465ea2", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0508 18:05:32.495827 3653 cni.go:480] [openshift-console/downloads-84f7f6d49-q2mf8 ddacbcd5c28a4892bfb8b4e51a089c4c627e050c12198dfb80c16357d0a99ce4 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-84f7f6d49-q2mf8 ddacbcd5c28a4892bfb8b4e51a089c4c627e050c12198dfb80c16357d0a99ce4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:2d:d0:95:76:f9\",\"name\":\"ddacbcd5c28a489\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c26adc83-58ba-4d8f-8699-1a40c7283d1a\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:05:41.295258 3653 base_network_controller_pods.go:489] [default/openshift-console/console-84579dd678-tz7bq] creating logical port openshift-console_console-84579dd678-tz7bq for pod on switch ip-10-0-0-68.ec2.internal I0508 18:05:41.295378 3653 kube.go:233] Patching annotations on pod openshift-console/console-84579dd678-tz7bq I0508 18:05:41.312119 3653 base_network_controller_pods.go:974] [default/openshift-console/console-84579dd678-tz7bq] addLogicalPort annotation time took 16.784217ms I0508 18:05:41.313488 3653 pods.go:270] [openshift-console/console-84579dd678-tz7bq] addLogicalPort took 18.248861ms, libovsdb time 970.254µs I0508 18:05:41.657448 3653 cni.go:454] [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key ] I0508 18:05:41.667238 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-console, podName: console-84579dd678-tz7bq, hostIfaceName: 3c8ca73530bcc1e, network: default, NAD default, SandboxID: "3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1", PCI device ID: , UID: "b20972c4-2e52-4cc9-82ba-3339c7f8f5d6", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0508 18:05:41.720098 3653 cni.go:480] [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:29:41:5c:99:17\",\"name\":\"3c8ca73530bcc1e\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/93d8bfe9-036e-4dd4-be42-5dc913e7f10a\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:05:45.712329 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0508 18:05:45.942853 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0508 18:05:48.008272 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:48.008292 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:48.008297 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:05:50.233126 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0508 18:05:50.288510 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-59b88794fc-jncfn I0508 18:05:50.288541 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-59b88794fc-jncfn W0508 18:05:51.951335 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0508 18:05:51.951363 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0508 18:05:52.732857 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0508 18:05:52.950665 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0508 18:05:53.762420 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0508 18:06:00.114338 3653 namespace.go:141] [openshift-kube-storage-version-migrator] updating namespace W0508 18:06:01.049655 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0508 18:06:01.049680 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0508 18:06:01.087714 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0508 18:06:01.087796 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0508 18:06:02.048299 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0508 18:06:02.777892 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0508 18:06:02.777905 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0508 18:06:02.777953 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0508 18:06:03.782417 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0508 18:06:05.007688 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0508 18:06:15.395026 3653 cni.go:454] [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key ] E0508 18:06:15.416010 3653 link_network_manager.go:101] Error while syncing link "3c8ca73530bcc1e": failed to get address from link "3c8ca73530bcc1e": failed to list addresses for "3c8ca73530bcc1e": no such device I0508 18:06:15.469148 3653 cni.go:480] [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-84579dd678-tz7bq 3c8ca73530bcc1efd76e3c97424dccc62fe4b1e8797fea194e9c00b7350b55d1 network default NAD default NAD key default], result "{}", err I0508 18:06:15.541174 3653 pods.go:175] Deleting pod: openshift-console/console-84579dd678-tz7bq I0508 18:06:15.542491 3653 pods.go:235] Attempting to release IPs for pod: openshift-console/console-84579dd678-tz7bq, ips: 10.128.0.21 I0508 18:06:15.552177 3653 base_network_controller_pods.go:489] [default/openshift-console/console-55cf6d5d8b-dgwlb] creating logical port openshift-console_console-55cf6d5d8b-dgwlb for pod on switch ip-10-0-0-68.ec2.internal I0508 18:06:15.552297 3653 kube.go:233] Patching annotations on pod openshift-console/console-55cf6d5d8b-dgwlb I0508 18:06:15.571384 3653 base_network_controller_pods.go:974] [default/openshift-console/console-55cf6d5d8b-dgwlb] addLogicalPort annotation time took 19.12702ms I0508 18:06:15.572657 3653 pods.go:270] [openshift-console/console-55cf6d5d8b-dgwlb] addLogicalPort took 20.496326ms, libovsdb time 800.191µs I0508 18:06:15.821856 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84579dd678-tz7bq I0508 18:06:15.821884 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84579dd678-tz7bq I0508 18:06:15.955945 3653 cni.go:454] [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key ] I0508 18:06:15.965791 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-console, podName: console-55cf6d5d8b-dgwlb, hostIfaceName: 92eee105b29b4f5, network: default, NAD default, SandboxID: "92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394", PCI device ID: , UID: "74c28929-9803-45fb-81f9-49d7937f9f74", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0508 18:06:16.015669 3653 cni.go:480] [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:b6:ba:54:bd:66\",\"name\":\"92eee105b29b4f5\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9fbf5071-443c-4ece-aa97-41e415c4a2a9\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:06:24.086428 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7fd96b5ff6-srxmg I0508 18:06:24.086453 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7fd96b5ff6-srxmg W0508 18:06:26.873368 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0508 18:06:27.104349 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84579dd678-8v857 I0508 18:06:27.104375 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84579dd678-8v857 I0508 18:06:34.086199 3653 cni.go:454] [openshift-multus/network-metrics-daemon-rdmhw 369b12af5b295164cff904b44d6fada1e544993b82c474dfca0d7d46a364cddd network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-rdmhw 369b12af5b295164cff904b44d6fada1e544993b82c474dfca0d7d46a364cddd network default NAD default NAD key ] I0508 18:06:34.095926 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-rdmhw, hostIfaceName: 369b12af5b29516, network: default, NAD default, SandboxID: "369b12af5b295164cff904b44d6fada1e544993b82c474dfca0d7d46a364cddd", PCI device ID: , UID: "8a089a1e-9b2d-46f9-a229-da6c4338ac86", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0508 18:06:34.144985 3653 cni.go:480] [openshift-multus/network-metrics-daemon-rdmhw 369b12af5b295164cff904b44d6fada1e544993b82c474dfca0d7d46a364cddd network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-rdmhw 369b12af5b295164cff904b44d6fada1e544993b82c474dfca0d7d46a364cddd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:03:37:11:80:13\",\"name\":\"369b12af5b29516\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/726f9849-5977-4745-954d-baa10943c905\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:06:39.221530 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0508 18:06:39.221561 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0508 18:06:41.009868 3653 base_network_controller_pods.go:489] [default/openshift-monitoring/telemeter-client-7b7568745f-6lhbv] creating logical port openshift-monitoring_telemeter-client-7b7568745f-6lhbv for pod on switch ip-10-0-0-68.ec2.internal I0508 18:06:41.009989 3653 kube.go:233] Patching annotations on pod openshift-monitoring/telemeter-client-7b7568745f-6lhbv I0508 18:06:41.030552 3653 base_network_controller_pods.go:974] [default/openshift-monitoring/telemeter-client-7b7568745f-6lhbv] addLogicalPort annotation time took 20.604124ms I0508 18:06:41.031781 3653 pods.go:270] [openshift-monitoring/telemeter-client-7b7568745f-6lhbv] addLogicalPort took 21.941407ms, libovsdb time 879.112µs I0508 18:06:41.442244 3653 cni.go:454] [openshift-monitoring/telemeter-client-7b7568745f-6lhbv 53f30a20ef302d517c7a89c9cde3f0b4a0090d0ced9e725fd352583be68b582f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-7b7568745f-6lhbv 53f30a20ef302d517c7a89c9cde3f0b4a0090d0ced9e725fd352583be68b582f network default NAD default NAD key ] I0508 18:06:41.451373 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-7b7568745f-6lhbv, hostIfaceName: 53f30a20ef302d5, network: default, NAD default, SandboxID: "53f30a20ef302d517c7a89c9cde3f0b4a0090d0ced9e725fd352583be68b582f", PCI device ID: , UID: "c207d76e-9290-45b6-b0e9-1bd7e1cac295", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0508 18:06:41.501412 3653 cni.go:480] [openshift-monitoring/telemeter-client-7b7568745f-6lhbv 53f30a20ef302d517c7a89c9cde3f0b4a0090d0ced9e725fd352583be68b582f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-7b7568745f-6lhbv 53f30a20ef302d517c7a89c9cde3f0b4a0090d0ced9e725fd352583be68b582f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:88:1f:43:84:96\",\"name\":\"53f30a20ef302d5\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f68bc73-9989-457c-a66a-c803c3c043ed\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:06:43.251282 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0508 18:06:43.251315 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0508 18:07:09.378325 3653 cni.go:454] [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key ] E0508 18:07:09.410757 3653 link_network_manager.go:101] Error while syncing link "92eee105b29b4f5": failed to get address from link "92eee105b29b4f5": failed to list addresses for "92eee105b29b4f5": no such device I0508 18:07:09.462559 3653 cni.go:480] [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-55cf6d5d8b-dgwlb 92eee105b29b4f55721ae0d59376c7a02fa9baf9d9d7184e53ed9d187e464394 network default NAD default NAD key default], result "{}", err I0508 18:07:09.507804 3653 pods.go:175] Deleting pod: openshift-console/console-55cf6d5d8b-dgwlb I0508 18:07:09.509107 3653 pods.go:235] Attempting to release IPs for pod: openshift-console/console-55cf6d5d8b-dgwlb, ips: 10.128.0.22 I0508 18:07:09.516232 3653 base_network_controller_pods.go:489] [default/openshift-console/console-7c5674588-xqh6f] creating logical port openshift-console_console-7c5674588-xqh6f for pod on switch ip-10-0-0-68.ec2.internal I0508 18:07:09.516406 3653 kube.go:233] Patching annotations on pod openshift-console/console-7c5674588-xqh6f I0508 18:07:09.535094 3653 base_network_controller_pods.go:974] [default/openshift-console/console-7c5674588-xqh6f] addLogicalPort annotation time took 18.727297ms I0508 18:07:09.536331 3653 pods.go:270] [openshift-console/console-7c5674588-xqh6f] addLogicalPort took 20.139596ms, libovsdb time 868.681µs I0508 18:07:09.872380 3653 cni.go:454] [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key ] I0508 18:07:09.882066 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-console, podName: console-7c5674588-xqh6f, hostIfaceName: 5b2e09a987e0762, network: default, NAD default, SandboxID: "5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857", PCI device ID: , UID: "94ec810c-0b1e-4440-a045-f14b412f2031", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0508 18:07:09.932471 3653 cni.go:480] [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:ac:d0:91:a4:43\",\"name\":\"5b2e09a987e0762\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79e6b58c-f7ff-44a5-a320-6d4491dac3c3\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:07:10.047701 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-55cf6d5d8b-dgwlb I0508 18:07:10.047743 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-55cf6d5d8b-dgwlb I0508 18:07:10.833272 3653 cni.go:454] [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key ] E0508 18:07:10.865771 3653 link_network_manager.go:101] Error while syncing link "3e632b88dcb4c64": failed to get address from link "3e632b88dcb4c64": failed to list addresses for "3e632b88dcb4c64": no such device I0508 18:07:10.912503 3653 cni.go:480] [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 3e632b88dcb4c64cda90c4af9ae6c82ca3da005109747a8b7eefd9dff104c8a4 network default NAD default NAD key default], result "{}", err I0508 18:07:10.969668 3653 pods.go:175] Deleting pod: openshift-monitoring/alertmanager-main-0 I0508 18:07:10.970988 3653 pods.go:235] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.128.0.18 I0508 18:07:11.077954 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0508 18:07:11.077983 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0508 18:07:11.119298 3653 base_network_controller_pods.go:489] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:07:11.119429 3653 kube.go:233] Patching annotations on pod openshift-monitoring/alertmanager-main-0 I0508 18:07:11.142313 3653 base_network_controller_pods.go:974] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 22.923921ms I0508 18:07:11.143581 3653 pods.go:270] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 24.30035ms, libovsdb time 891.862µs I0508 18:07:11.478316 3653 cni.go:454] [openshift-monitoring/alertmanager-main-0 3cac2e10aa03ec31d3c70a844769f0dc6b47d298f684147c2c134b55a5962ee6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 3cac2e10aa03ec31d3c70a844769f0dc6b47d298f684147c2c134b55a5962ee6 network default NAD default NAD key ] I0508 18:07:11.487921 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 3cac2e10aa03ec3, network: default, NAD default, SandboxID: "3cac2e10aa03ec31d3c70a844769f0dc6b47d298f684147c2c134b55a5962ee6", PCI device ID: , UID: "2c7187b5-e784-4d39-9286-76be46bbeda1", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0508 18:07:11.538712 3653 cni.go:480] [openshift-monitoring/alertmanager-main-0 3cac2e10aa03ec31d3c70a844769f0dc6b47d298f684147c2c134b55a5962ee6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 3cac2e10aa03ec31d3c70a844769f0dc6b47d298f684147c2c134b55a5962ee6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:ab:2a:c6:b1:ba\",\"name\":\"3cac2e10aa03ec3\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5669fc27-28b7-48d0-89d6-c99e98b6767b\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:07:20.105633 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0508 18:07:21.437888 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-55cf6d5d8b-4svwj I0508 18:07:21.437921 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-55cf6d5d8b-4svwj W0508 18:07:41.532390 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:07:41.532413 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0508 18:07:41.532418 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0508 18:07:46.434080 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0508 18:07:46.434110 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0508 18:08:15.387880 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressIP" totalItems=5 I0508 18:08:35.169319 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=72 I0508 18:08:43.411836 3653 cni.go:454] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key ] E0508 18:08:43.443256 3653 link_network_manager.go:101] Error while syncing link "17e3916cf33417d": failed to get address from link "17e3916cf33417d": failed to list addresses for "17e3916cf33417d": no such device I0508 18:08:43.495094 3653 cni.go:480] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 17e3916cf33417d6943cecdc804881dc0a280a34d1aa3e25513f92af55254f7e network default NAD default NAD key default], result "{}", err I0508 18:08:43.537314 3653 pods.go:175] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 I0508 18:08:43.538557 3653 pods.go:235] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89, ips: 10.128.0.8 I0508 18:08:45.586730 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressService" totalItems=6 I0508 18:08:51.168757 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=15 I0508 18:08:58.169442 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=7 I0508 18:09:25.628376 3653 cni.go:454] [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key ] E0508 18:09:25.659885 3653 link_network_manager.go:101] Error while syncing link "5b2e09a987e0762": failed to get address from link "5b2e09a987e0762": failed to list addresses for "5b2e09a987e0762": no such device I0508 18:09:25.710969 3653 cni.go:480] [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7c5674588-xqh6f 5b2e09a987e0762a701775dfaeadd79b6264d5a0df186cb09f8e8bf61d9bd857 network default NAD default NAD key default], result "{}", err I0508 18:09:25.761367 3653 pods.go:175] Deleting pod: openshift-console/console-7c5674588-xqh6f I0508 18:09:25.762769 3653 pods.go:235] Attempting to release IPs for pod: openshift-console/console-7c5674588-xqh6f, ips: 10.128.0.24 I0508 18:09:25.770392 3653 base_network_controller_pods.go:489] [default/openshift-console/console-7964f866f7-75t2f] creating logical port openshift-console_console-7964f866f7-75t2f for pod on switch ip-10-0-0-68.ec2.internal I0508 18:09:25.770500 3653 kube.go:233] Patching annotations on pod openshift-console/console-7964f866f7-75t2f I0508 18:09:25.787534 3653 base_network_controller_pods.go:974] [default/openshift-console/console-7964f866f7-75t2f] addLogicalPort annotation time took 17.069253ms I0508 18:09:25.788726 3653 pods.go:270] [openshift-console/console-7964f866f7-75t2f] addLogicalPort took 18.344464ms, libovsdb time 844.597µs I0508 18:09:26.131123 3653 cni.go:454] [openshift-console/console-7964f866f7-75t2f fd2f4d333c2a8354c6add9220b7b1b4297375449899085071ac4c49720a244b8 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7964f866f7-75t2f fd2f4d333c2a8354c6add9220b7b1b4297375449899085071ac4c49720a244b8 network default NAD default NAD key ] I0508 18:09:26.140378 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-console, podName: console-7964f866f7-75t2f, hostIfaceName: fd2f4d333c2a835, network: default, NAD default, SandboxID: "fd2f4d333c2a8354c6add9220b7b1b4297375449899085071ac4c49720a244b8", PCI device ID: , UID: "81d774bf-6824-4e2e-ba4d-932df23805af", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0508 18:09:26.189466 3653 cni.go:480] [openshift-console/console-7964f866f7-75t2f fd2f4d333c2a8354c6add9220b7b1b4297375449899085071ac4c49720a244b8 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7964f866f7-75t2f fd2f4d333c2a8354c6add9220b7b1b4297375449899085071ac4c49720a244b8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:2f:02:40:df:55\",\"name\":\"fd2f4d333c2a835\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be43c17f-3e8e-43b1-9d2e-fea2ccf340fb\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:09:26.575593 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7c5674588-xqh6f I0508 18:09:26.575622 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7c5674588-xqh6f W0508 18:09:36.628917 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0508 18:09:36.862380 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7c5674588-2w8jm I0508 18:09:36.862410 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7c5674588-2w8jm I0508 18:09:44.304639 3653 base_network_controller_pods.go:489] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-4bvlw for pod on switch ip-10-0-0-68.ec2.internal I0508 18:09:44.304791 3653 kube.go:233] Patching annotations on pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw I0508 18:09:44.318333 3653 base_network_controller_pods.go:974] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw] addLogicalPort annotation time took 13.580172ms I0508 18:09:44.319665 3653 pods.go:270] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw] addLogicalPort took 15.04107ms, libovsdb time 979.393µs I0508 18:09:44.665121 3653 cni.go:454] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw b2c49c0ed0206104563191171228c0969666434cc84a553b787d1a15af518b1d network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw b2c49c0ed0206104563191171228c0969666434cc84a553b787d1a15af518b1d network default NAD default NAD key ] I0508 18:09:44.674132 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-4bvlw, hostIfaceName: b2c49c0ed020610, network: default, NAD default, SandboxID: "b2c49c0ed0206104563191171228c0969666434cc84a553b787d1a15af518b1d", PCI device ID: , UID: "d94dd7b4-4876-4c37-9ce2-e553a1952886", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0508 18:09:44.723961 3653 cni.go:480] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw b2c49c0ed0206104563191171228c0969666434cc84a553b787d1a15af518b1d network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-4bvlw b2c49c0ed0206104563191171228c0969666434cc84a553b787d1a15af518b1d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:47:6c:50:3e:ea\",\"name\":\"b2c49c0ed020610\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67d2794a-b924-4a49-9459-d63e415065a9\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:09:47.678574 3653 util.go:863] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0508 18:09:55.850659 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.ClusterUserDefinedNetwork" totalItems=7 W0508 18:09:58.686420 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0508 18:09:58.727030 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0508 18:09:58.727100 3653 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-9hrxx IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-9hrxx: no pod IPs found I0508 18:10:00.177021 3653 base_network_controller_pods.go:489] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29637730-gf8gg for pod on switch ip-10-0-0-68.ec2.internal I0508 18:10:00.177151 3653 kube.go:233] Patching annotations on pod openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg I0508 18:10:00.190667 3653 base_network_controller_pods.go:974] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg] addLogicalPort annotation time took 13.551142ms I0508 18:10:00.191846 3653 pods.go:270] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg] addLogicalPort took 14.842881ms, libovsdb time 852.998µs I0508 18:10:00.553640 3653 cni.go:454] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key ] I0508 18:10:00.562608 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29637730-gf8gg, hostIfaceName: a0caa4aef795733, network: default, NAD default, SandboxID: "a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc", PCI device ID: , UID: "d012ac2b-180b-4f4c-8f84-a3e0710133a8", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0508 18:10:00.613629 3653 cni.go:480] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:0b:b5:4e:95:06\",\"name\":\"a0caa4aef795733\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8135fd30-f466-414a-962c-b4fa58041757\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:10:03.749387 3653 cni.go:454] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key ] E0508 18:10:03.781416 3653 link_network_manager.go:101] Error while syncing link "a0caa4aef795733": failed to get address from link "a0caa4aef795733": failed to list addresses for "a0caa4aef795733": no such device I0508 18:10:03.832966 3653 cni.go:480] [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg a0caa4aef795733e8f6daed4b125ef0f2f66eafcd3cf12298c4ea7a28578cadc network default NAD default NAD key default], result "{}", err I0508 18:10:03.874905 3653 pods.go:175] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg I0508 18:10:03.876271 3653 pods.go:235] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg, ips: 10.128.0.28 I0508 18:10:47.169775 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=71 I0508 18:10:57.169338 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=145 I0508 18:10:59.169034 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=707 I0508 18:10:59.475134 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressFirewall" totalItems=9 I0508 18:11:17.872695 3653 namespace.go:97] [tsf] adding namespace I0508 18:11:17.873647 3653 namespace.go:101] [tsf] adding namespace took 926.26µs I0508 18:11:17.889034 3653 namespace.go:141] [tsf] updating namespace I0508 18:11:17.904686 3653 namespace.go:141] [tsf] updating namespace I0508 18:11:21.769391 3653 namespace.go:97] [rhbk-operator] adding namespace I0508 18:11:21.769485 3653 namespace.go:97] [tsf-keycloak] adding namespace I0508 18:11:21.770302 3653 namespace.go:101] [rhbk-operator] adding namespace took 886.458µs I0508 18:11:21.771026 3653 namespace.go:101] [tsf-keycloak] adding namespace took 1.523286ms I0508 18:11:21.772266 3653 namespace.go:97] [tsf-tas] adding namespace I0508 18:11:21.772877 3653 namespace.go:101] [tsf-tas] adding namespace took 596.102µs I0508 18:11:21.774827 3653 namespace.go:97] [konflux-ui] adding namespace I0508 18:11:21.774871 3653 namespace.go:97] [rhtpa-operator] adding namespace I0508 18:11:21.774991 3653 namespace.go:97] [konflux-operator] adding namespace I0508 18:11:21.775043 3653 namespace.go:97] [cert-manager-operator] adding namespace I0508 18:11:21.775052 3653 namespace.go:97] [tsf-tpa] adding namespace I0508 18:11:21.775523 3653 namespace.go:101] [konflux-ui] adding namespace took 679.409µs I0508 18:11:21.776036 3653 namespace.go:101] [rhtpa-operator] adding namespace took 1.145024ms I0508 18:11:21.776596 3653 namespace.go:101] [konflux-operator] adding namespace took 1.589761ms I0508 18:11:21.777200 3653 namespace.go:101] [cert-manager-operator] adding namespace took 2.135709ms I0508 18:11:21.777815 3653 namespace.go:101] [tsf-tpa] adding namespace took 2.748552ms I0508 18:11:21.787728 3653 namespace.go:141] [rhbk-operator] updating namespace I0508 18:11:21.801813 3653 namespace.go:141] [rhbk-operator] updating namespace I0508 18:11:21.818404 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:11:21.825951 3653 namespace.go:141] [tsf-tas] updating namespace I0508 18:11:21.826030 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:11:21.826134 3653 namespace.go:141] [rhtpa-operator] updating namespace I0508 18:11:21.839102 3653 namespace.go:141] [konflux-operator] updating namespace I0508 18:11:21.844997 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:11:21.848874 3653 namespace.go:141] [cert-manager-operator] updating namespace I0508 18:11:21.848919 3653 namespace.go:141] [rhbk-operator] updating namespace I0508 18:11:21.850534 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:11:21.855416 3653 namespace.go:141] [tsf-tas] updating namespace I0508 18:11:21.874712 3653 namespace.go:141] [tsf-tas] updating namespace I0508 18:11:21.894702 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:11:21.914752 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:11:21.937801 3653 namespace.go:141] [rhtpa-operator] updating namespace I0508 18:11:21.953464 3653 namespace.go:141] [rhtpa-operator] updating namespace I0508 18:11:21.958433 3653 namespace.go:141] [konflux-operator] updating namespace I0508 18:11:21.982524 3653 namespace.go:141] [konflux-operator] updating namespace I0508 18:11:22.010276 3653 namespace.go:141] [cert-manager-operator] updating namespace I0508 18:11:22.023178 3653 namespace.go:141] [cert-manager-operator] updating namespace I0508 18:11:22.037785 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:11:22.067244 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:11:22.176625 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:11:22.851161 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:11:22.861977 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:11:23.027939 3653 base_network_controller_pods.go:489] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-0-68.ec2.internal I0508 18:11:23.028072 3653 kube.go:233] Patching annotations on pod tsf/test-tsf-subscriptions I0508 18:11:23.047625 3653 base_network_controller_pods.go:974] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 19.594999ms I0508 18:11:23.048809 3653 pods.go:270] [tsf/test-tsf-subscriptions] addLogicalPort took 20.886673ms, libovsdb time 845.449µs I0508 18:11:23.395764 3653 cni.go:454] [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key ] I0508 18:11:23.404710 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: b14892a6a70148a, network: default, NAD default, SandboxID: "b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed", PCI device ID: , UID: "8bfc3e64-c43f-402b-ac93-11c712114660", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0508 18:11:23.455639 3653 cni.go:480] [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:9d:7a:0c:e8:a9\",\"name\":\"b14892a6a70148a\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24980940-3c84-4a4e-b6f0-9feff085de3c\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:11:29.443015 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0508 18:11:29.443046 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0508 18:11:30.455522 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0508 18:11:30.455544 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0508 18:11:32.456434 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0508 18:11:32.456458 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0508 18:11:33.477905 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0508 18:11:33.477924 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0508 18:11:35.558355 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0508 18:11:35.558376 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0508 18:11:37.503122 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0508 18:11:37.503147 3653 base_network_controller_policy.go:656] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0508 18:11:45.710888 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0508 18:11:46.773634 3653 namespace.go:97] [cert-manager] adding namespace I0508 18:11:46.774498 3653 namespace.go:101] [cert-manager] adding namespace took 838.384µs I0508 18:11:46.802584 3653 namespace.go:141] [cert-manager] updating namespace I0508 18:11:46.820855 3653 namespace.go:141] [cert-manager] updating namespace W0508 18:11:52.462107 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0508 18:11:52.462128 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0508 18:11:55.482370 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressQoS" totalItems=11 I0508 18:11:56.272712 3653 namespace.go:141] [openshift-operators] updating namespace I0508 18:11:56.290455 3653 namespace.go:141] [openshift-operators] updating namespace W0508 18:11:58.542242 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0508 18:11:59.616181 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0508 18:11:59.892283 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0508 18:11:59.923136 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0508 18:12:02.320572 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0508 18:12:06.720074 3653 namespace.go:97] [trusted-artifact-signer] adding namespace I0508 18:12:06.720974 3653 namespace.go:101] [trusted-artifact-signer] adding namespace took 877.554µs I0508 18:12:06.835367 3653 namespace.go:141] [trusted-artifact-signer] updating namespace W0508 18:12:06.890375 3653 util.go:863] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0508 18:12:06.893588 3653 namespace.go:141] [trusted-artifact-signer] updating namespace I0508 18:12:09.016285 3653 namespace.go:141] [openshift-operators] updating namespace I0508 18:12:09.540517 3653 base_network_controller_pods.go:489] [default/openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-gzv5h for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:09.540647 3653 kube.go:233] Patching annotations on pod openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h I0508 18:12:09.555900 3653 base_network_controller_pods.go:974] [default/openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h] addLogicalPort annotation time took 15.291842ms I0508 18:12:09.557033 3653 pods.go:270] [openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h] addLogicalPort took 16.539205ms, libovsdb time 787.204µs I0508 18:12:11.710817 3653 cni.go:454] [openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h a9e62975e48559dacb2c3abb6bae78f828b62b4851d757d211f7182493786435 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h a9e62975e48559dacb2c3abb6bae78f828b62b4851d757d211f7182493786435 network default NAD default NAD key ] I0508 18:12:11.720131 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-gzv5h, hostIfaceName: a9e62975e48559d, network: default, NAD default, SandboxID: "a9e62975e48559dacb2c3abb6bae78f828b62b4851d757d211f7182493786435", PCI device ID: , UID: "ef1b0b06-175b-403d-a60d-9415bb08efb0", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0508 18:12:11.770156 3653 cni.go:480] [openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h a9e62975e48559dacb2c3abb6bae78f828b62b4851d757d211f7182493786435 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-gzv5h a9e62975e48559dacb2c3abb6bae78f828b62b4851d757d211f7182493786435 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:0e:f4:93:ac:71\",\"name\":\"a9e62975e48559d\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/47ef5ffb-6daa-4552-9409-caaf4dbdebc5\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:12:22.798790 3653 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0508 18:12:24.525361 3653 namespace.go:97] [openshift-pipelines] adding namespace I0508 18:12:24.526242 3653 namespace.go:101] [openshift-pipelines] adding namespace took 857.028µs I0508 18:12:24.541495 3653 namespace.go:141] [openshift-pipelines] updating namespace I0508 18:12:25.232169 3653 namespace.go:141] [cert-manager] updating namespace I0508 18:12:25.433247 3653 namespace.go:141] [cert-manager-operator] updating namespace I0508 18:12:25.836433 3653 namespace.go:141] [konflux-operator] updating namespace I0508 18:12:26.036817 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:12:26.237810 3653 namespace.go:141] [open-cluster-management-2q5t7gpgv0osum5rqnh4kbqvj22icgps] updating namespace I0508 18:12:26.638593 3653 namespace.go:141] [rhbk-operator] updating namespace I0508 18:12:26.839666 3653 namespace.go:141] [rhtpa-operator] updating namespace I0508 18:12:27.040416 3653 namespace.go:141] [trusted-artifact-signer] updating namespace I0508 18:12:27.177400 3653 namespace.go:141] [tsf] updating namespace I0508 18:12:27.382180 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:12:27.582522 3653 namespace.go:141] [tsf-tas] updating namespace I0508 18:12:27.782064 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:12:27.917125 3653 namespace.go:141] [cert-manager] updating namespace I0508 18:12:27.928456 3653 namespace.go:141] [cert-manager-operator] updating namespace I0508 18:12:27.936719 3653 namespace.go:141] [default] updating namespace I0508 18:12:27.945446 3653 namespace.go:141] [konflux-operator] updating namespace I0508 18:12:27.956692 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:12:27.966398 3653 namespace.go:141] [open-cluster-management-2q5t7gpgv0osum5rqnh4kbqvj22icgps] updating namespace I0508 18:12:27.976120 3653 namespace.go:141] [openshift] updating namespace I0508 18:12:27.986314 3653 namespace.go:141] [rhbk-operator] updating namespace I0508 18:12:27.996337 3653 namespace.go:141] [rhtpa-operator] updating namespace I0508 18:12:28.003992 3653 namespace.go:141] [trusted-artifact-signer] updating namespace I0508 18:12:28.013413 3653 namespace.go:141] [tsf] updating namespace I0508 18:12:28.023262 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:12:28.044664 3653 namespace.go:141] [tsf-tas] updating namespace I0508 18:12:28.066566 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:12:28.166232 3653 namespace.go:141] [cert-manager] updating namespace I0508 18:12:28.267409 3653 namespace.go:141] [cert-manager-operator] updating namespace I0508 18:12:28.364152 3653 namespace.go:141] [default] updating namespace I0508 18:12:28.466450 3653 namespace.go:141] [konflux-operator] updating namespace I0508 18:12:28.564733 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:12:28.665041 3653 namespace.go:141] [open-cluster-management-2q5t7gpgv0osum5rqnh4kbqvj22icgps] updating namespace I0508 18:12:28.764946 3653 namespace.go:141] [openshift] updating namespace I0508 18:12:28.863724 3653 namespace.go:141] [rhbk-operator] updating namespace I0508 18:12:28.967767 3653 namespace.go:141] [rhtpa-operator] updating namespace I0508 18:12:29.064824 3653 namespace.go:141] [trusted-artifact-signer] updating namespace I0508 18:12:29.163336 3653 namespace.go:141] [tsf] updating namespace I0508 18:12:29.264815 3653 namespace.go:141] [tsf-keycloak] updating namespace I0508 18:12:29.302687 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0508 18:12:29.365285 3653 namespace.go:141] [tsf-tas] updating namespace I0508 18:12:29.464524 3653 namespace.go:141] [tsf-tpa] updating namespace I0508 18:12:29.759146 3653 namespace.go:141] [openshift-pipelines] updating namespace I0508 18:12:36.399481 3653 cni.go:454] [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key ] E0508 18:12:36.421461 3653 link_network_manager.go:101] Error while syncing link "b14892a6a70148a": failed to get address from link "b14892a6a70148a": failed to list addresses for "b14892a6a70148a": no such device I0508 18:12:36.475958 3653 cni.go:480] [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions b14892a6a70148a613fafb8c2de10797720bbad33cd5adae7ded0dbf098ba1ed network default NAD default NAD key default], result "{}", err I0508 18:12:36.529177 3653 pods.go:175] Deleting pod: tsf/test-tsf-subscriptions I0508 18:12:36.530617 3653 pods.go:235] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.29 I0508 18:12:36.560591 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0508 18:12:36.560621 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0508 18:12:37.565886 3653 base_network_controller_pods.go:489] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-gvt4c for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:37.565985 3653 kube.go:233] Patching annotations on pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c I0508 18:12:37.584953 3653 base_network_controller_pods.go:974] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c] addLogicalPort annotation time took 19.000961ms I0508 18:12:37.586268 3653 pods.go:270] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c] addLogicalPort took 20.395786ms, libovsdb time 920.424µs I0508 18:12:37.667704 3653 base_network_controller_pods.go:489] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-rsjh4 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:37.667839 3653 kube.go:233] Patching annotations on pod openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4 I0508 18:12:37.687433 3653 base_network_controller_pods.go:974] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4] addLogicalPort annotation time took 19.6457ms I0508 18:12:37.688656 3653 pods.go:270] [openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4] addLogicalPort took 20.97297ms, libovsdb time 838.632µs I0508 18:12:37.783197 3653 base_network_controller_pods.go:489] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:37.783311 3653 kube.go:233] Patching annotations on pod openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb I0508 18:12:37.804642 3653 base_network_controller_pods.go:974] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb] addLogicalPort annotation time took 21.364547ms I0508 18:12:37.805917 3653 pods.go:270] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb] addLogicalPort took 22.736168ms, libovsdb time 903.582µs I0508 18:12:37.907162 3653 base_network_controller_pods.go:489] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:37.907310 3653 kube.go:233] Patching annotations on pod tsf/test-tsf-infrastructure I0508 18:12:37.925318 3653 base_network_controller_pods.go:974] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 18.039743ms I0508 18:12:37.926511 3653 pods.go:270] [tsf/test-tsf-infrastructure] addLogicalPort took 19.363824ms, libovsdb time 801.525µs I0508 18:12:37.930746 3653 cni.go:454] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c b06244f0a2dc06cfd723e792a8b3b4302c375dd20166475acd6b5b46b0425507 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c b06244f0a2dc06cfd723e792a8b3b4302c375dd20166475acd6b5b46b0425507 network default NAD default NAD key ] I0508 18:12:37.940349 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-gvt4c, hostIfaceName: b06244f0a2dc06c, network: default, NAD default, SandboxID: "b06244f0a2dc06cfd723e792a8b3b4302c375dd20166475acd6b5b46b0425507", PCI device ID: , UID: "b6d63775-a0c2-4eb0-87fb-8ef464d68591", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0508 18:12:37.991024 3653 cni.go:480] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c b06244f0a2dc06cfd723e792a8b3b4302c375dd20166475acd6b5b46b0425507 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-gvt4c b06244f0a2dc06cfd723e792a8b3b4302c375dd20166475acd6b5b46b0425507 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:22:5a:87:0d:85\",\"name\":\"b06244f0a2dc06c\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b91aafe-a33c-4258-9aed-785e90eec9ff\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:12:38.031006 3653 cni.go:454] [openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4 d92642f911dd50e7048afdb224ee0efc66be54d4743d2ac8b0537dd3eaed0aae network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4 d92642f911dd50e7048afdb224ee0efc66be54d4743d2ac8b0537dd3eaed0aae network default NAD default NAD key ] I0508 18:12:38.041060 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-rsjh4, hostIfaceName: d92642f911dd50e, network: default, NAD default, SandboxID: "d92642f911dd50e7048afdb224ee0efc66be54d4743d2ac8b0537dd3eaed0aae", PCI device ID: , UID: "dcc75c04-35b4-46e7-ba6a-d904f809e71c", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0508 18:12:38.091192 3653 cni.go:480] [openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4 d92642f911dd50e7048afdb224ee0efc66be54d4743d2ac8b0537dd3eaed0aae network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-rsjh4 d92642f911dd50e7048afdb224ee0efc66be54d4743d2ac8b0537dd3eaed0aae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:ab:42:f0:cd:a0\",\"name\":\"d92642f911dd50e\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f8d579b-bf06-4058-8bba-fc6360d6bd14\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:12:38.147410 3653 cni.go:454] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb f479b3d0ebe2dc1b6ff4a9375c26ea39d81310d585fa7dcc828f801ca5228d3e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb f479b3d0ebe2dc1b6ff4a9375c26ea39d81310d585fa7dcc828f801ca5228d3e network default NAD default NAD key ] I0508 18:12:38.156896 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb, hostIfaceName: f479b3d0ebe2dc1, network: default, NAD default, SandboxID: "f479b3d0ebe2dc1b6ff4a9375c26ea39d81310d585fa7dcc828f801ca5228d3e", PCI device ID: , UID: "74b250db-1a4c-44a4-b7fe-af200608f7ab", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0508 18:12:38.206155 3653 cni.go:480] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb f479b3d0ebe2dc1b6ff4a9375c26ea39d81310d585fa7dcc828f801ca5228d3e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-bllpb f479b3d0ebe2dc1b6ff4a9375c26ea39d81310d585fa7dcc828f801ca5228d3e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:2c:00:ff:1f:b2\",\"name\":\"f479b3d0ebe2dc1\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d7c6010a-4c93-4036-88c2-2112f2ccdd84\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:12:38.270024 3653 cni.go:454] [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key ] I0508 18:12:38.280148 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: cc0c3664d313ba7, network: default, NAD default, SandboxID: "cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42", PCI device ID: , UID: "2b497275-ca60-44f2-a999-1ab7b0c6dd38", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0508 18:12:38.330462 3653 cni.go:480] [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:e9:c5:fe:ef:97\",\"name\":\"cc0c3664d313ba7\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e867a8ff-ae83-4f4b-b503-a8122f5e42ae\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:12:40.782796 3653 base_network_controller_pods.go:489] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-qj827 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:40.782903 3653 kube.go:233] Patching annotations on pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827 I0508 18:12:40.795716 3653 base_network_controller_pods.go:489] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-pzpfk for pod on switch ip-10-0-0-68.ec2.internal I0508 18:12:40.795836 3653 kube.go:233] Patching annotations on pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk I0508 18:12:40.800263 3653 base_network_controller_pods.go:974] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827] addLogicalPort annotation time took 17.394708ms I0508 18:12:40.801809 3653 pods.go:270] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827] addLogicalPort took 19.027815ms, libovsdb time 1.017226ms I0508 18:12:40.809521 3653 base_network_controller_pods.go:974] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk] addLogicalPort annotation time took 13.729029ms I0508 18:12:40.810896 3653 pods.go:270] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk] addLogicalPort took 15.201829ms, libovsdb time 894.344µs W0508 18:12:42.440111 3653 util.go:863] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0508 18:12:42.440137 3653 util.go:863] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0508 18:12:42.440145 3653 util.go:863] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0508 18:12:42.468154 3653 util.go:863] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0508 18:12:42.468179 3653 util.go:863] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0508 18:12:42.468187 3653 util.go:863] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0508 18:12:42.757313 3653 util.go:863] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0508 18:12:42.757331 3653 util.go:863] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0508 18:12:42.757337 3653 util.go:863] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0508 18:12:42.757343 3653 util.go:863] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0508 18:12:44.805100 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.UserDefinedNetwork" totalItems=10 I0508 18:12:47.453337 3653 cni.go:454] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827 15d5d03311c9f1713b3a1ccd47d794966b1bc6587e1d6fbc2d2dff2523a7eca1 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827 15d5d03311c9f1713b3a1ccd47d794966b1bc6587e1d6fbc2d2dff2523a7eca1 network default NAD default NAD key ] I0508 18:12:47.461253 3653 cni.go:454] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk 8e175bba16acd5c0c72234e7c4130ecb7fd4c90a0d30776ea1ed6c4f840f5f7d network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk 8e175bba16acd5c0c72234e7c4130ecb7fd4c90a0d30776ea1ed6c4f840f5f7d network default NAD default NAD key ] I0508 18:12:47.462948 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-qj827, hostIfaceName: 15d5d03311c9f17, network: default, NAD default, SandboxID: "15d5d03311c9f1713b3a1ccd47d794966b1bc6587e1d6fbc2d2dff2523a7eca1", PCI device ID: , UID: "39724506-5c6d-484e-a94c-253da261b311", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0508 18:12:47.472642 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-pzpfk, hostIfaceName: 8e175bba16acd5c, network: default, NAD default, SandboxID: "8e175bba16acd5c0c72234e7c4130ecb7fd4c90a0d30776ea1ed6c4f840f5f7d", PCI device ID: , UID: "8b9e6407-0997-4770-badc-fab4c0da687e", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0508 18:12:47.517877 3653 cni.go:480] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827 15d5d03311c9f1713b3a1ccd47d794966b1bc6587e1d6fbc2d2dff2523a7eca1 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-qj827 15d5d03311c9f1713b3a1ccd47d794966b1bc6587e1d6fbc2d2dff2523a7eca1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:d5:e5:0e:23:23\",\"name\":\"15d5d03311c9f17\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9063364f-25be-4cb5-821b-e46e8dc7239c\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:12:47.541270 3653 cni.go:480] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk 8e175bba16acd5c0c72234e7c4130ecb7fd4c90a0d30776ea1ed6c4f840f5f7d network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-pzpfk 8e175bba16acd5c0c72234e7c4130ecb7fd4c90a0d30776ea1ed6c4f840f5f7d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:88:37:c2:71:4d\",\"name\":\"8e175bba16acd5c\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8301dbe9-b691-40f5-9649-67d3d4687add\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:12:48.481529 3653 util.go:863] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0508 18:12:48.505192 3653 util.go:863] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0508 18:12:53.456854 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0508 18:12:53.456876 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0508 18:12:53.456882 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0508 18:12:53.478659 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0508 18:12:53.478681 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0508 18:12:53.478686 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0508 18:13:05.871576 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0508 18:13:09.484174 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0508 18:13:09.505488 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0508 18:13:12.604313 3653 cni.go:454] [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key ] E0508 18:13:12.638954 3653 link_network_manager.go:101] Error while syncing link "cc0c3664d313ba7": failed to get address from link "cc0c3664d313ba7": failed to list addresses for "cc0c3664d313ba7": no such device I0508 18:13:12.696495 3653 cni.go:480] [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure cc0c3664d313ba7d240f9029ba27c96b17a6c5e8bca0227dcc9421fce1c53d42 network default NAD default NAD key default], result "{}", err I0508 18:13:12.753420 3653 pods.go:175] Deleting pod: tsf/test-tsf-infrastructure I0508 18:13:12.754897 3653 pods.go:235] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.34 I0508 18:13:12.780824 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0508 18:13:12.780853 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0508 18:13:14.315880 3653 base_network_controller_pods.go:489] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-0-68.ec2.internal I0508 18:13:14.315987 3653 kube.go:233] Patching annotations on pod tsf/test-tsf-iam I0508 18:13:14.332233 3653 base_network_controller_pods.go:974] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 16.262411ms I0508 18:13:14.334414 3653 pods.go:270] [tsf/test-tsf-iam] addLogicalPort took 18.54777ms, libovsdb time 1.279675ms I0508 18:13:14.739695 3653 cni.go:454] [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key ] I0508 18:13:14.749359 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 6d41a3e74be69c1, network: default, NAD default, SandboxID: "6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae", PCI device ID: , UID: "c1d6a6ef-e174-4c36-8296-69e5f22f15a6", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0508 18:13:14.800060 3653 cni.go:480] [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:fd:24:5c:83:82\",\"name\":\"6d41a3e74be69c1\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/828943eb-53ba-4fe1-a270-e2cded679535\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:13:15.168681 3653 base_network_controller_policy.go:1227] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0508 18:13:15.169347 3653 base_network_controller_policy.go:1102] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [default-network-controller_tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0508 18:13:15.171647 3653 base_network_controller_policy.go:1280] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0508 18:13:21.958782 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0508 18:13:21.958814 3653 util.go:863] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0508 18:13:21.959262 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0508 18:13:21.959289 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak I0508 18:13:23.152928 3653 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0508 18:13:44.596606 3653 base_network_controller_pods.go:489] [default/openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9] creating logical port openshift-pipelines_tekton-results-api-f744cfc9f-s2pp9 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:13:44.596742 3653 kube.go:233] Patching annotations on pod openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9 I0508 18:13:44.613860 3653 base_network_controller_pods.go:974] [default/openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9] addLogicalPort annotation time took 17.14788ms I0508 18:13:44.615104 3653 pods.go:270] [openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9] addLogicalPort took 18.516917ms, libovsdb time 851.553µs I0508 18:13:44.958635 3653 cni.go:454] [openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9 3ad36e48a009fc8eda28242d7e6c19ddb22c85c80bb7b3777ab942bd451dca4a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9 3ad36e48a009fc8eda28242d7e6c19ddb22c85c80bb7b3777ab942bd451dca4a network default NAD default NAD key ] I0508 18:13:44.968058 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-f744cfc9f-s2pp9, hostIfaceName: 3ad36e48a009fc8, network: default, NAD default, SandboxID: "3ad36e48a009fc8eda28242d7e6c19ddb22c85c80bb7b3777ab942bd451dca4a", PCI device ID: , UID: "c3247694-42e2-4d84-afb3-dfed1325ceb1", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0508 18:13:45.020908 3653 cni.go:480] [openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9 3ad36e48a009fc8eda28242d7e6c19ddb22c85c80bb7b3777ab942bd451dca4a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-api-f744cfc9f-s2pp9 3ad36e48a009fc8eda28242d7e6c19ddb22c85c80bb7b3777ab942bd451dca4a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:62:7d:a8:a7:8a\",\"name\":\"3ad36e48a009fc8\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/93267121-db4d-4fde-950c-d47a6fa6d10f\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:13:51.814419 3653 util.go:863] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0508 18:13:51.814441 3653 util.go:863] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0508 18:13:51.814448 3653 util.go:863] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0508 18:14:04.973341 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0508 18:14:04.973362 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0508 18:14:04.973368 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0508 18:14:05.885711 3653 cni.go:454] [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key ] E0508 18:14:05.918341 3653 link_network_manager.go:101] Error while syncing link "6d41a3e74be69c1": failed to get address from link "6d41a3e74be69c1": failed to list addresses for "6d41a3e74be69c1": no such device I0508 18:14:05.960762 3653 cni.go:480] [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam 6d41a3e74be69c1ce4f2b66e879405a8e2d509ad44b4f92f0981f449bd70b1ae network default NAD default NAD key default], result "{}", err I0508 18:14:06.011144 3653 pods.go:175] Deleting pod: tsf/test-tsf-iam I0508 18:14:06.012746 3653 pods.go:235] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.37 I0508 18:14:06.057791 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0508 18:14:06.057819 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0508 18:14:06.884676 3653 base_network_controller_pods.go:489] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:06.884782 3653 kube.go:233] Patching annotations on pod tsf-tas/test-tsf-tas I0508 18:14:06.901592 3653 base_network_controller_pods.go:974] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 16.842232ms I0508 18:14:06.902982 3653 pods.go:270] [tsf-tas/test-tsf-tas] addLogicalPort took 18.319583ms, libovsdb time 944.453µs I0508 18:14:07.244650 3653 cni.go:454] [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key ] I0508 18:14:07.254888 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 992600f6594ee7d, network: default, NAD default, SandboxID: "992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53", PCI device ID: , UID: "1bbdcb27-7a33-4ee6-aa2f-bec4b54c8f51", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0508 18:14:07.307244 3653 cni.go:480] [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:53:39:b9:7a:82\",\"name\":\"992600f6594ee7d\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a0dac62-12e9-45c7-9946-e5a6bdafe112\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:14:07.907941 3653 base_network_controller_pods.go:489] [default/tsf-tas/segment-backup-installation-ksxlc-h5qgj] creating logical port tsf-tas_segment-backup-installation-ksxlc-h5qgj for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:07.908099 3653 kube.go:233] Patching annotations on pod tsf-tas/segment-backup-installation-ksxlc-h5qgj I0508 18:14:07.921914 3653 base_network_controller_pods.go:974] [default/tsf-tas/segment-backup-installation-ksxlc-h5qgj] addLogicalPort annotation time took 13.846741ms I0508 18:14:07.923286 3653 pods.go:270] [tsf-tas/segment-backup-installation-ksxlc-h5qgj] addLogicalPort took 15.360581ms, libovsdb time 904.029µs I0508 18:14:08.270488 3653 cni.go:454] [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key ] I0508 18:14:08.280167 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-ksxlc-h5qgj, hostIfaceName: 751719c221a1141, network: default, NAD default, SandboxID: "751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082", PCI device ID: , UID: "ddf81095-aa39-4e85-a91d-195dc0e6dc64", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0508 18:14:08.335709 3653 cni.go:480] [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:12:2b:71:ec:d0\",\"name\":\"751719c221a1141\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4dd3ffe7-a1b2-4f1a-a23c-9973c01af17e\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:14:08.557047 3653 base_network_controller_pods.go:489] [default/tsf-tas/fulcio-server-bc8f86b58-zlmc7] creating logical port tsf-tas_fulcio-server-bc8f86b58-zlmc7 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:08.557193 3653 kube.go:233] Patching annotations on pod tsf-tas/fulcio-server-bc8f86b58-zlmc7 I0508 18:14:08.575842 3653 base_network_controller_pods.go:974] [default/tsf-tas/fulcio-server-bc8f86b58-zlmc7] addLogicalPort annotation time took 18.686473ms I0508 18:14:08.577063 3653 pods.go:270] [tsf-tas/fulcio-server-bc8f86b58-zlmc7] addLogicalPort took 20.036244ms, libovsdb time 849.124µs I0508 18:14:08.730615 3653 base_network_controller_pods.go:489] [default/tsf-tas/ctlog-createtree-job-7dzrc-ghpc9] creating logical port tsf-tas_ctlog-createtree-job-7dzrc-ghpc9 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:08.730714 3653 kube.go:233] Patching annotations on pod tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 I0508 18:14:08.750291 3653 base_network_controller_pods.go:974] [default/tsf-tas/ctlog-createtree-job-7dzrc-ghpc9] addLogicalPort annotation time took 19.609949ms I0508 18:14:08.751482 3653 pods.go:270] [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9] addLogicalPort took 20.882041ms, libovsdb time 775.348µs I0508 18:14:09.049846 3653 base_network_controller_pods.go:489] [default/tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp] creating logical port tsf-tas_trillian-logserver-7c4d8d56cc-j7jcp for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:09.049950 3653 kube.go:233] Patching annotations on pod tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp I0508 18:14:09.067297 3653 base_network_controller_pods.go:974] [default/tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp] addLogicalPort annotation time took 17.380712ms I0508 18:14:09.068595 3653 pods.go:270] [tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp] addLogicalPort took 18.764097ms, libovsdb time 839.48µs I0508 18:14:09.092015 3653 cni.go:454] [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key ] I0508 18:14:09.101698 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-7dzrc-ghpc9, hostIfaceName: 3ad4391b9efce2c, network: default, NAD default, SandboxID: "3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217", PCI device ID: , UID: "cb5978ab-a1d3-45f9-9524-1d0125714c32", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0508 18:14:09.152796 3653 cni.go:480] [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:7b:f7:91:7f:be\",\"name\":\"3ad4391b9efce2c\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19c512f2-ced6-42a2-968e-3a341b37ffad\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:14:09.221537 3653 cni.go:454] [tsf-tas/fulcio-server-bc8f86b58-zlmc7 0992dd9ab606c86de0743f79c1f053b6c253d6dd3b7ef2e428a9a4c5a090a970 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-bc8f86b58-zlmc7 0992dd9ab606c86de0743f79c1f053b6c253d6dd3b7ef2e428a9a4c5a090a970 network default NAD default NAD key ] I0508 18:14:09.231248 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-bc8f86b58-zlmc7, hostIfaceName: 0992dd9ab606c86, network: default, NAD default, SandboxID: "0992dd9ab606c86de0743f79c1f053b6c253d6dd3b7ef2e428a9a4c5a090a970", PCI device ID: , UID: "d869cad2-5e21-46c8-ae26-4f7dea66d33c", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0508 18:14:09.273484 3653 base_network_controller_pods.go:489] [default/tsf-tas/rekor-createtree-job-qbs8s-5cf7t] creating logical port tsf-tas_rekor-createtree-job-qbs8s-5cf7t for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:09.273578 3653 kube.go:233] Patching annotations on pod tsf-tas/rekor-createtree-job-qbs8s-5cf7t I0508 18:14:09.283555 3653 cni.go:480] [tsf-tas/fulcio-server-bc8f86b58-zlmc7 0992dd9ab606c86de0743f79c1f053b6c253d6dd3b7ef2e428a9a4c5a090a970 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-bc8f86b58-zlmc7 0992dd9ab606c86de0743f79c1f053b6c253d6dd3b7ef2e428a9a4c5a090a970 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:34:8f:2d:71:18\",\"name\":\"0992dd9ab606c86\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0000851-2894-4d11-bcec-ad98c1e3b87d\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:14:09.291960 3653 base_network_controller_pods.go:974] [default/tsf-tas/rekor-createtree-job-qbs8s-5cf7t] addLogicalPort annotation time took 18.412101ms I0508 18:14:09.293233 3653 pods.go:270] [tsf-tas/rekor-createtree-job-qbs8s-5cf7t] addLogicalPort took 19.763619ms, libovsdb time 779.315µs I0508 18:14:09.950357 3653 cni.go:454] [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key ] I0508 18:14:09.962686 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-qbs8s-5cf7t, hostIfaceName: c79439667ed531e, network: default, NAD default, SandboxID: "c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806", PCI device ID: , UID: "b242540b-9034-4656-94cd-c4b2d7771029", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0508 18:14:10.040057 3653 cni.go:454] [tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp 69576f882be0cdf7ef44dc96c9388cda8807bc4ebdf5f20e6711720a2bc3abd6 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp 69576f882be0cdf7ef44dc96c9388cda8807bc4ebdf5f20e6711720a2bc3abd6 network default NAD default NAD key ] I0508 18:14:10.047932 3653 cni.go:480] [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:79:0f:54:01:d7\",\"name\":\"c79439667ed531e\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/53a6c0a1-4114-47a2-b1c9-ac4d22e357c3\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:14:10.052816 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-7c4d8d56cc-j7jcp, hostIfaceName: 69576f882be0cdf, network: default, NAD default, SandboxID: "69576f882be0cdf7ef44dc96c9388cda8807bc4ebdf5f20e6711720a2bc3abd6", PCI device ID: , UID: "3328bcfc-c7b4-4c74-9615-0903721e420c", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0508 18:14:10.128932 3653 cni.go:480] [tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp 69576f882be0cdf7ef44dc96c9388cda8807bc4ebdf5f20e6711720a2bc3abd6 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-7c4d8d56cc-j7jcp 69576f882be0cdf7ef44dc96c9388cda8807bc4ebdf5f20e6711720a2bc3abd6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:90:87:5f:4f:1e\",\"name\":\"69576f882be0cdf\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/25ce9e18-ddd0-4262-b78b-e41b43fcaada\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:14:13.944649 3653 util.go:863] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0508 18:14:13.944679 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0508 18:14:13.972807 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0508 18:14:13.972905 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0508 18:14:17.398850 3653 util.go:863] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0508 18:14:22.424761 3653 util.go:863] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0508 18:14:23.024289 3653 util.go:863] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0508 18:14:25.043654 3653 base_network_controller_pods.go:489] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-htn2t for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:25.043787 3653 kube.go:233] Patching annotations on pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t I0508 18:14:25.058128 3653 base_network_controller_pods.go:974] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t] addLogicalPort annotation time took 14.390335ms I0508 18:14:25.059407 3653 pods.go:270] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t] addLogicalPort took 15.773793ms, libovsdb time 850.465µs I0508 18:14:25.410578 3653 cni.go:454] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t 6822fea8cab83eb55112feafd042806cc76d21a35fbd5cc16f153ae114793525 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t 6822fea8cab83eb55112feafd042806cc76d21a35fbd5cc16f153ae114793525 network default NAD default NAD key ] I0508 18:14:25.419873 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-htn2t, hostIfaceName: 6822fea8cab83eb, network: default, NAD default, SandboxID: "6822fea8cab83eb55112feafd042806cc76d21a35fbd5cc16f153ae114793525", PCI device ID: , UID: "bf2a9501-5bf2-4b45-b6ea-8d6c435ddba5", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0508 18:14:25.473587 3653 cni.go:480] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t 6822fea8cab83eb55112feafd042806cc76d21a35fbd5cc16f153ae114793525 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-htn2t 6822fea8cab83eb55112feafd042806cc76d21a35fbd5cc16f153ae114793525 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:78:95:fa:56:32\",\"name\":\"6822fea8cab83eb\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/14393aba-784f-438b-ac68-eb59c7f72fe2\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:14:26.019086 3653 cni.go:454] [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key ] E0508 18:14:26.050136 3653 link_network_manager.go:101] Error while syncing link "751719c221a1141": failed to get address from link "751719c221a1141": failed to list addresses for "751719c221a1141": no such device E0508 18:14:26.050348 3653 link_network_manager.go:101] Error while syncing link "751719c221a1141": failed to get address from link "751719c221a1141": failed to list addresses for "751719c221a1141": no such device I0508 18:14:26.091018 3653 cni.go:480] [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/segment-backup-installation-ksxlc-h5qgj 751719c221a1141bc2783926830a91fc232a15c81a85e2a82e92986d4dd6d082 network default NAD default NAD key default], result "{}", err I0508 18:14:26.161274 3653 pods.go:175] Deleting pod: tsf-tas/segment-backup-installation-ksxlc-h5qgj I0508 18:14:26.162770 3653 pods.go:235] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-ksxlc-h5qgj, ips: 10.128.0.40 W0508 18:14:28.033165 3653 util.go:863] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0508 18:14:30.401514 3653 util.go:863] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0508 18:14:30.401536 3653 util.go:863] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0508 18:14:39.043932 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0508 18:14:40.203800 3653 base_network_controller_pods.go:489] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-pljrf for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:40.203934 3653 kube.go:233] Patching annotations on pod openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf I0508 18:14:40.222255 3653 base_network_controller_pods.go:974] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf] addLogicalPort annotation time took 18.354243ms I0508 18:14:40.223497 3653 pods.go:270] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf] addLogicalPort took 19.720185ms, libovsdb time 845.092µs I0508 18:14:40.563342 3653 cni.go:454] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf 4791810d25e697d4dc78fef8fcb2e9d6ba65eb12771baf46d640935d843c2c5a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf 4791810d25e697d4dc78fef8fcb2e9d6ba65eb12771baf46d640935d843c2c5a network default NAD default NAD key ] I0508 18:14:40.572486 3653 helper_linux.go:546] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-pljrf, hostIfaceName: 4791810d25e697d, network: default, NAD default, SandboxID: "4791810d25e697d4dc78fef8fcb2e9d6ba65eb12771baf46d640935d843c2c5a", PCI device ID: , UID: "dd538e40-87cc-442e-8138-c2a0515b1b8b", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0508 18:14:40.624692 3653 cni.go:480] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf 4791810d25e697d4dc78fef8fcb2e9d6ba65eb12771baf46d640935d843c2c5a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-pljrf 4791810d25e697d4dc78fef8fcb2e9d6ba65eb12771baf46d640935d843c2c5a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:a7:c2:2e:83:d7\",\"name\":\"4791810d25e697d\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/effeb18e-c7fe-40ad-b29a-5205032420a5\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:14:51.116755 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0508 18:14:56.210287 3653 cni.go:454] [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key ] E0508 18:14:56.246069 3653 link_network_manager.go:101] Error while syncing link "992600f6594ee7d": failed to get address from link "992600f6594ee7d": failed to list addresses for "992600f6594ee7d": no such device I0508 18:14:56.301157 3653 cni.go:480] [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas 992600f6594ee7d753529758324cea5e143ebe8e0e3bfeac0800a958e479cd53 network default NAD default NAD key default], result "{}", err I0508 18:14:56.518249 3653 pods.go:175] Deleting pod: tsf-tas/test-tsf-tas I0508 18:14:56.519714 3653 pods.go:235] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.39 I0508 18:14:56.545178 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0508 18:14:56.545206 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0508 18:14:57.244013 3653 base_network_controller_pods.go:489] [default/tsf/patch-tekton-config-hx5qt] creating logical port tsf_patch-tekton-config-hx5qt for pod on switch ip-10-0-0-68.ec2.internal I0508 18:14:57.244117 3653 kube.go:233] Patching annotations on pod tsf/patch-tekton-config-hx5qt I0508 18:14:57.257430 3653 base_network_controller_pods.go:974] [default/tsf/patch-tekton-config-hx5qt] addLogicalPort annotation time took 13.347582ms I0508 18:14:57.258705 3653 pods.go:270] [tsf/patch-tekton-config-hx5qt] addLogicalPort took 14.70849ms, libovsdb time 855.727µs I0508 18:14:57.602498 3653 cni.go:454] [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key ] ADD starting CNI request [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key ] I0508 18:14:57.612305 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-hx5qt, hostIfaceName: 99da4ca7d8d1512, network: default, NAD default, SandboxID: "99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2", PCI device ID: , UID: "0bc152b3-5c52-44be-a91b-ee82c30ff2e5", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0508 18:14:57.665305 3653 cni.go:480] [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key default] ADD finished CNI request [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:f1:04:65:50:e1\",\"name\":\"99da4ca7d8d1512\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a275e1d-3955-43fa-ad9c-666a1c69557b\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:15:06.273414 3653 cni.go:454] [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key ] DEL starting CNI request [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key ] E0508 18:15:06.309393 3653 link_network_manager.go:101] Error while syncing link "99da4ca7d8d1512": failed to get address from link "99da4ca7d8d1512": failed to list addresses for "99da4ca7d8d1512": no such device I0508 18:15:06.369781 3653 cni.go:480] [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key default] DEL finished CNI request [tsf/patch-tekton-config-hx5qt 99da4ca7d8d1512616bc235fdcf809bfe89e99b196553cf79028925dbbc204f2 network default NAD default NAD key default], result "{}", err I0508 18:15:06.419462 3653 pods.go:175] Deleting pod: tsf/patch-tekton-config-hx5qt I0508 18:15:06.420874 3653 pods.go:235] Attempting to release IPs for pod: tsf/patch-tekton-config-hx5qt, ips: 10.128.0.47 I0508 18:15:07.539629 3653 base_network_controller_pods.go:489] [default/tsf/tsf-tekton-configuration-vfrrm] creating logical port tsf_tsf-tekton-configuration-vfrrm for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:07.539746 3653 kube.go:233] Patching annotations on pod tsf/tsf-tekton-configuration-vfrrm I0508 18:15:07.550861 3653 base_network_controller_pods.go:974] [default/tsf/tsf-tekton-configuration-vfrrm] addLogicalPort annotation time took 11.157945ms I0508 18:15:07.552326 3653 pods.go:270] [tsf/tsf-tekton-configuration-vfrrm] addLogicalPort took 12.717295ms, libovsdb time 989.129µs I0508 18:15:07.899800 3653 cni.go:454] [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key ] I0508 18:15:07.909370 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-vfrrm, hostIfaceName: 8272eaff6b8a6bc, network: default, NAD default, SandboxID: "8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca", PCI device ID: , UID: "61f8ceaa-f346-41e5-8557-12c89fbf1c05", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0508 18:15:07.963390 3653 cni.go:480] [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:fa:0e:c4:74:cd\",\"name\":\"8272eaff6b8a6bc\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29d7b651-bcbc-4feb-9e13-628b4beeae2d\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:15:14.322529 3653 cni.go:454] [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key ] E0508 18:15:14.357317 3653 link_network_manager.go:101] Error while syncing link "c79439667ed531e": failed to get address from link "c79439667ed531e": failed to list addresses for "c79439667ed531e": no such device I0508 18:15:14.411953 3653 cni.go:480] [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-qbs8s-5cf7t c79439667ed531e1b1b2329d4f0fe69efe4fc789854b4a54ee4c1452016c6806 network default NAD default NAD key default], result "{}", err I0508 18:15:14.460622 3653 pods.go:175] Deleting pod: tsf-tas/rekor-createtree-job-qbs8s-5cf7t I0508 18:15:14.462044 3653 pods.go:235] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-qbs8s-5cf7t, ips: 10.128.0.44 I0508 18:15:15.328326 3653 cni.go:454] [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key ] E0508 18:15:15.362648 3653 link_network_manager.go:101] Error while syncing link "8272eaff6b8a6bc": failed to get address from link "8272eaff6b8a6bc": failed to list addresses for "8272eaff6b8a6bc": no such device I0508 18:15:15.415331 3653 cni.go:480] [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-vfrrm 8272eaff6b8a6bce1d1d87a2cf0738e4ae145b68906cc302dab4151be24db0ca network default NAD default NAD key default], result "{}", err I0508 18:15:15.461723 3653 pods.go:175] Deleting pod: tsf/tsf-tekton-configuration-vfrrm I0508 18:15:15.463226 3653 pods.go:235] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-vfrrm, ips: 10.128.0.48 I0508 18:15:16.520118 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-hx5qt I0508 18:15:16.520157 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-hx5qt I0508 18:15:17.172650 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=97 I0508 18:15:17.302006 3653 base_network_controller_pods.go:489] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:17.302121 3653 kube.go:233] Patching annotations on pod tsf-tpa/pre-install-tsf-tpa I0508 18:15:17.315511 3653 base_network_controller_pods.go:974] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 13.421268ms I0508 18:15:17.316917 3653 pods.go:270] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 14.93025ms, libovsdb time 880.734µs I0508 18:15:17.662045 3653 cni.go:454] [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key ] I0508 18:15:17.671353 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 2e18a2b5d793b86, network: default, NAD default, SandboxID: "2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd", PCI device ID: , UID: "c798c345-4433-4ee3-bab9-5f0febfa5cce", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0508 18:15:17.723987 3653 cni.go:480] [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:d8:e6:40:a3:5e\",\"name\":\"2e18a2b5d793b86\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/764e8513-0a81-4b2d-adb5-a62daa490b95\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:15:19.354498 3653 cni.go:454] [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key ] E0508 18:15:19.388792 3653 link_network_manager.go:101] Error while syncing link "3ad4391b9efce2c": failed to get address from link "3ad4391b9efce2c": failed to list addresses for "3ad4391b9efce2c": no such device I0508 18:15:19.441200 3653 cni.go:480] [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 3ad4391b9efce2c764b0aa28e094e1447f49ec5797ecacb758fc88b53ae74217 network default NAD default NAD key default], result "{}", err I0508 18:15:19.504035 3653 pods.go:175] Deleting pod: tsf-tas/ctlog-createtree-job-7dzrc-ghpc9 I0508 18:15:19.505365 3653 pods.go:235] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-7dzrc-ghpc9, ips: 10.128.0.42 I0508 18:15:20.860708 3653 base_network_controller_pods.go:489] [default/tsf-tas/rekor-search-ui-5d99664696-nf6pr] creating logical port tsf-tas_rekor-search-ui-5d99664696-nf6pr for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:20.860846 3653 kube.go:233] Patching annotations on pod tsf-tas/rekor-search-ui-5d99664696-nf6pr I0508 18:15:20.874665 3653 base_network_controller_pods.go:974] [default/tsf-tas/rekor-search-ui-5d99664696-nf6pr] addLogicalPort annotation time took 13.852758ms I0508 18:15:20.876003 3653 pods.go:270] [tsf-tas/rekor-search-ui-5d99664696-nf6pr] addLogicalPort took 15.314844ms, libovsdb time 895.009µs I0508 18:15:21.223501 3653 cni.go:454] [tsf-tas/rekor-search-ui-5d99664696-nf6pr 6a0bb68736e92bac1e91a8b7bf4f2320971987d66fe2d622c53d3b7504ce5381 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-5d99664696-nf6pr 6a0bb68736e92bac1e91a8b7bf4f2320971987d66fe2d622c53d3b7504ce5381 network default NAD default NAD key ] I0508 18:15:21.232726 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-5d99664696-nf6pr, hostIfaceName: 6a0bb68736e92ba, network: default, NAD default, SandboxID: "6a0bb68736e92bac1e91a8b7bf4f2320971987d66fe2d622c53d3b7504ce5381", PCI device ID: , UID: "c3b3cd4d-b867-4330-a7d9-dd759d200567", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0508 18:15:21.286472 3653 cni.go:480] [tsf-tas/rekor-search-ui-5d99664696-nf6pr 6a0bb68736e92bac1e91a8b7bf4f2320971987d66fe2d622c53d3b7504ce5381 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-5d99664696-nf6pr 6a0bb68736e92bac1e91a8b7bf4f2320971987d66fe2d622c53d3b7504ce5381 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:43:f7:34:35:8e\",\"name\":\"6a0bb68736e92ba\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/272ec552-9f57-459f-90fd-2d8661907732\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:15:21.369523 3653 cni.go:454] [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key ] E0508 18:15:21.403568 3653 link_network_manager.go:101] Error while syncing link "2e18a2b5d793b86": failed to get address from link "2e18a2b5d793b86": failed to list addresses for "2e18a2b5d793b86": no such device I0508 18:15:21.460627 3653 cni.go:480] [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 2e18a2b5d793b86a6382367b4d7e77376b51a4e62f8c01d96231bb110d2c42dd network default NAD default NAD key default], result "{}", err I0508 18:15:21.504706 3653 pods.go:175] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0508 18:15:21.506055 3653 pods.go:235] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.49 I0508 18:15:21.533552 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0508 18:15:21.533579 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0508 18:15:21.737762 3653 base_network_controller_pods.go:489] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:21.737870 3653 kube.go:233] Patching annotations on pod tsf-tpa/test-tsf-tpa I0508 18:15:21.751132 3653 base_network_controller_pods.go:974] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.298347ms I0508 18:15:21.752449 3653 pods.go:270] [tsf-tpa/test-tsf-tpa] addLogicalPort took 14.703319ms, libovsdb time 907.417µs I0508 18:15:22.134789 3653 cni.go:454] [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key ] I0508 18:15:22.146048 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: a0b3a7ab7d7555a, network: default, NAD default, SandboxID: "a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54", PCI device ID: , UID: "e5d34f20-4782-4a81-b8b1-5beb4edf298b", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0508 18:15:22.210116 3653 cni.go:480] [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:5d:c8:c4:cc:0a\",\"name\":\"a0b3a7ab7d7555a\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/43067bb7-a4bc-4d72-8b62-c22c45132584\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:15:22.459350 3653 namespace.go:141] [openshift-pipelines] updating namespace I0508 18:15:23.428517 3653 base_network_controller_pods.go:489] [default/tsf-tas/rekor-server-84f6c6ffb4-2x2nb] creating logical port tsf-tas_rekor-server-84f6c6ffb4-2x2nb for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:23.428666 3653 kube.go:233] Patching annotations on pod tsf-tas/rekor-server-84f6c6ffb4-2x2nb I0508 18:15:23.443076 3653 base_network_controller_pods.go:974] [default/tsf-tas/rekor-server-84f6c6ffb4-2x2nb] addLogicalPort annotation time took 14.446616ms I0508 18:15:23.444607 3653 pods.go:270] [tsf-tas/rekor-server-84f6c6ffb4-2x2nb] addLogicalPort took 16.11585ms, libovsdb time 1.015348ms W0508 18:15:23.683507 3653 util.go:863] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0508 18:15:24.387073 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0508 18:15:27.791761 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0508 18:15:30.095885 3653 cni.go:454] [tsf-tas/rekor-server-84f6c6ffb4-2x2nb 4d4a52dd79d60124a06463489f147f4e91d55f9e815f5a8a77a89d105510909f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-84f6c6ffb4-2x2nb 4d4a52dd79d60124a06463489f147f4e91d55f9e815f5a8a77a89d105510909f network default NAD default NAD key ] I0508 18:15:30.105112 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-84f6c6ffb4-2x2nb, hostIfaceName: 4d4a52dd79d6012, network: default, NAD default, SandboxID: "4d4a52dd79d60124a06463489f147f4e91d55f9e815f5a8a77a89d105510909f", PCI device ID: , UID: "37e6fcf6-820d-49ca-94e9-af9d3108d7ff", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0508 18:15:30.157267 3653 cni.go:480] [tsf-tas/rekor-server-84f6c6ffb4-2x2nb 4d4a52dd79d60124a06463489f147f4e91d55f9e815f5a8a77a89d105510909f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-84f6c6ffb4-2x2nb 4d4a52dd79d60124a06463489f147f4e91d55f9e815f5a8a77a89d105510909f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:23:c6:e7:d2:fa\",\"name\":\"4d4a52dd79d6012\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4dd25101-7997-4ceb-a0bc-7842fde20c1f\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:15:32.443756 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0508 18:15:36.218472 3653 base_network_controller_pods.go:489] [default/tsf-tpa/server-fd96dd64c-t7jgv] creating logical port tsf-tpa_server-fd96dd64c-t7jgv for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:36.218596 3653 kube.go:233] Patching annotations on pod tsf-tpa/server-fd96dd64c-t7jgv I0508 18:15:36.232933 3653 base_network_controller_pods.go:974] [default/tsf-tpa/server-fd96dd64c-t7jgv] addLogicalPort annotation time took 14.371538ms I0508 18:15:36.234419 3653 pods.go:270] [tsf-tpa/server-fd96dd64c-t7jgv] addLogicalPort took 15.966814ms, libovsdb time 967.939µs W0508 18:15:36.447668 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0508 18:15:38.023357 3653 base_network_controller_pods.go:489] [default/tsf-tpa/importer-86bcdf477b-6prn2] creating logical port tsf-tpa_importer-86bcdf477b-6prn2 for pod on switch ip-10-0-0-68.ec2.internal I0508 18:15:38.023463 3653 kube.go:233] Patching annotations on pod tsf-tpa/importer-86bcdf477b-6prn2 I0508 18:15:38.037817 3653 base_network_controller_pods.go:974] [default/tsf-tpa/importer-86bcdf477b-6prn2] addLogicalPort annotation time took 14.39253ms I0508 18:15:38.039081 3653 pods.go:270] [tsf-tpa/importer-86bcdf477b-6prn2] addLogicalPort took 15.740515ms, libovsdb time 832.427µs I0508 18:15:41.415170 3653 cni.go:454] [tsf-tpa/server-fd96dd64c-t7jgv c0f59744d2929dc085ad312dae6b814f90ee929de306d045c5459977b1335015 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-fd96dd64c-t7jgv c0f59744d2929dc085ad312dae6b814f90ee929de306d045c5459977b1335015 network default NAD default NAD key ] I0508 18:15:41.424373 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tpa, podName: server-fd96dd64c-t7jgv, hostIfaceName: c0f59744d2929dc, network: default, NAD default, SandboxID: "c0f59744d2929dc085ad312dae6b814f90ee929de306d045c5459977b1335015", PCI device ID: , UID: "b6011103-6a7f-461b-9304-3b1c2e3c3d9e", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0508 18:15:41.479184 3653 cni.go:480] [tsf-tpa/server-fd96dd64c-t7jgv c0f59744d2929dc085ad312dae6b814f90ee929de306d045c5459977b1335015 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-fd96dd64c-t7jgv c0f59744d2929dc085ad312dae6b814f90ee929de306d045c5459977b1335015 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:d4:0a:d6:54:fa\",\"name\":\"c0f59744d2929dc\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/61230afb-a5d9-4477-a14b-bd63b34cccec\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0508 18:15:42.008619 3653 cni.go:454] [tsf-tpa/importer-86bcdf477b-6prn2 033c2b80a84714c505882ffc9bc141afee72ca034c737283dac1fee581b4dc7f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-6prn2 033c2b80a84714c505882ffc9bc141afee72ca034c737283dac1fee581b4dc7f network default NAD default NAD key ] I0508 18:15:42.018132 3653 helper_linux.go:546] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-6prn2, hostIfaceName: 033c2b80a84714c, network: default, NAD default, SandboxID: "033c2b80a84714c505882ffc9bc141afee72ca034c737283dac1fee581b4dc7f", PCI device ID: , UID: "78157a5f-eec5-4590-8468-fc71b9e485c2", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0508 18:15:42.070443 3653 cni.go:480] [tsf-tpa/importer-86bcdf477b-6prn2 033c2b80a84714c505882ffc9bc141afee72ca034c737283dac1fee581b4dc7f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-6prn2 033c2b80a84714c505882ffc9bc141afee72ca034c737283dac1fee581b4dc7f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:4b:3f:9c:10:5a\",\"name\":\"033c2b80a84714c\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b6aa146-e8d0-4613-acff-9011c59a54ee\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0508 18:15:43.450285 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server W0508 18:15:44.544476 3653 util.go:863] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0508 18:15:55.554689 3653 util.go:952] Failed to build node endpoints for node ip-10-0-0-68.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0508 18:16:12.686205 3653 cni.go:454] [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key ] E0508 18:16:12.708144 3653 link_network_manager.go:101] Error while syncing link "a0b3a7ab7d7555a": failed to get address from link "a0b3a7ab7d7555a": failed to list addresses for "a0b3a7ab7d7555a": no such device I0508 18:16:12.766949 3653 cni.go:480] [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa a0b3a7ab7d7555a367dba0370c5f8875bb3e2c6cfb4a2dc3aa422899b61b4f54 network default NAD default NAD key default], result "{}", err I0508 18:16:12.820666 3653 pods.go:175] Deleting pod: tsf-tpa/test-tsf-tpa I0508 18:16:12.822042 3653 pods.go:235] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.51 I0508 18:16:12.851571 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0508 18:16:12.851594 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0508 18:16:13.658942 3653 namespace.go:97] [image-controller] adding namespace I0508 18:16:13.660004 3653 namespace.go:101] [image-controller] adding namespace took 1.042466ms I0508 18:16:13.661430 3653 namespace.go:97] [integration-service] adding namespace I0508 18:16:13.662093 3653 namespace.go:101] [integration-service] adding namespace took 648.999µs I0508 18:16:13.663395 3653 namespace.go:97] [build-service] adding namespace I0508 18:16:13.663968 3653 namespace.go:101] [build-service] adding namespace took 560.155µs I0508 18:16:13.666062 3653 namespace.go:97] [default-tenant] adding namespace I0508 18:16:13.666663 3653 namespace.go:101] [default-tenant] adding namespace took 589.365µs I0508 18:16:13.721569 3653 namespace.go:141] [image-controller] updating namespace I0508 18:16:13.756308 3653 namespace.go:141] [image-controller] updating namespace I0508 18:16:13.786542 3653 namespace.go:141] [integration-service] updating namespace I0508 18:16:13.837497 3653 namespace.go:141] [integration-service] updating namespace I0508 18:16:13.899580 3653 namespace.go:141] [build-service] updating namespace I0508 18:16:13.943365 3653 namespace.go:141] [build-service] updating namespace I0508 18:16:14.025595 3653 namespace.go:141] [build-service] updating namespace I0508 18:16:14.025675 3653 namespace.go:141] [default-tenant] updating namespace I0508 18:16:14.025723 3653 namespace.go:141] [integration-service] updating namespace I0508 18:16:14.049828 3653 namespace.go:141] [default-tenant] updating namespace I0508 18:16:14.062261 3653 namespace.go:97] [release-service] adding namespace I0508 18:16:14.063047 3653 namespace.go:101] [release-service] adding namespace took 765.301µs I0508 18:16:14.091749 3653 namespace.go:141] [release-service] updating namespace I0508 18:16:14.127866 3653 namespace.go:141] [release-service] updating namespace I0508 18:16:14.129530 3653 namespace.go:141] [konflux-ui] updating namespace I0508 18:16:14.255630 3653 namespace.go:97] [konflux-info] adding namespace I0508 18:16:14.256441 3653 namespace.go:101] [konflux-info] adding namespace took 790.404µs I0508 18:16:14.435751 3653 namespace.go:97] [namespace-lister] adding namespace I0508 18:16:14.436704 3653 namespace.go:101] [namespace-lister] adding namespace took 931.218µs I0508 18:16:14.549587 3653 namespace.go:141] [konflux-info] updating namespace I0508 18:16:14.687724 3653 namespace.go:141] [konflux-info] updating namespace I0508 18:16:14.738628 3653 namespace.go:97] [enterprise-contract-service] adding namespace I0508 18:16:14.739489 3653 namespace.go:101] [enterprise-contract-service] adding namespace took 839.109µs I0508 18:16:14.807632 3653 namespace.go:141] [image-controller] updating namespace I0508 18:16:14.845664 3653 namespace.go:141] [namespace-lister] updating namespace I0508 18:16:14.970275 3653 namespace.go:141] [namespace-lister] updating namespace I0508 18:16:15.164747 3653 namespace.go:141] [default-tenant] updating namespace I0508 18:16:15.182521 3653 namespace.go:97] [segment-bridge] adding namespace I0508 18:16:15.183371 3653 namespace.go:101] [segment-bridge] adding namespace took 832.519µs I0508 18:16:15.198782 3653 namespace.go:141] [enterprise-contract-service] updating namespace I0508 18:16:15.223681 3653 base_network_controller_policy.go:1227] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0508 18:16:15.224145 3653 base_network_controller_policy.go:1102] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [default-network-controller_namespace-lister_LS{ML:{app: proxy,},}] I0508 18:16:15.226765 3653 base_network_controller_policy.go:1280] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0508 18:16:15.266882 3653 base_network_controller_policy.go:1227] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0508 18:16:15.266905 3653 base_network_controller_policy.go:1102] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0508 18:16:15.267604 3653 base_network_controller_policy.go:1280] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0508 18:16:15.291828 3653 namespace.go:97] [konflux-cli] adding namespace I0508 18:16:15.292611 3653 namespace.go:101] [konflux-cli] adding namespace took 746.835µs I0508 18:16:15.559455 3653 namespace.go:141] [enterprise-contract-service] updating namespace I0508 18:16:15.626365 3653 namespace.go:141] [segment-bridge] updating namespace I0508 18:16:15.703086 3653 namespace.go:141] [segment-bridge] updating namespace I0508 18:16:15.832818 3653 namespace.go:141] [konflux-cli] updating namespace I0508 18:16:15.945634 3653 namespace.go:141] [konflux-cli] updating namespace I0508 18:16:16.591613 3653 namespace.go:141] [build-service] updating namespace I0508 18:16:17.403074 3653 namespace.go:141] [default-tenant] updating namespace I0508 18:16:17.663954 3653 namespace.go:141] [enterprise-contract-service] updating namespace I0508 18:16:17.841724 3653 namespace.go:141] [image-controller] updating namespace I0508 18:16:18.040924 3653 namespace.go:141] [integration-service] updating namespace I0508 18:16:18.273907 3653 namespace.go:141] [konflux-cli] updating namespace I0508 18:16:18.487704 3653 namespace.go:141] [konflux-info] updating namespace I0508 18:16:18.736321 3653 namespace.go:141] [namespace-lister] updating namespace I0508 18:16:18.873460 3653 namespace.go:141] [release-service] updating namespace I0508 18:16:19.075887 3653 namespace.go:141] [segment-bridge] updating namespace I0508 18:16:19.190523 3653 namespace.go:141] [build-service] updating namespace I0508 18:16:19.206695 3653 namespace.go:141] [default-tenant] updating namespace I0508 18:16:19.222674 3653 namespace.go:141] [enterprise-contract-service] updating namespace I0508 18:16:19.235906 3653 namespace.go:141] [image-controller] updating namespace I0508 18:16:19.248758 3653 namespace.go:141] [integration-service] updating namespace I0508 18:16:19.272860 3653 namespace.go:141] [konflux-cli] updating namespace I0508 18:16:19.292685 3653 namespace.go:141] [konflux-info] updating namespace I0508 18:16:19.313699 3653 namespace.go:141] [namespace-lister] updating namespace I0508 18:16:19.325703 3653 namespace.go:141] [release-service] updating namespace I0508 18:16:19.340556 3653 namespace.go:141] [segment-bridge] updating namespace I0508 18:16:19.389845 3653 namespace.go:141] [build-service] updating namespace I0508 18:16:19.451049 3653 namespace.go:141] [default-tenant] updating namespace I0508 18:16:19.523950 3653 namespace.go:141] [enterprise-contract-service] updating namespace I0508 18:16:19.618099 3653 namespace.go:141] [image-controller] updating namespace I0508 18:16:19.726467 3653 namespace.go:141] [integration-service] updating namespace I0508 18:16:19.822795 3653 namespace.go:141] [konflux-cli] updating namespace I0508 18:16:19.931087 3653 namespace.go:141] [konflux-info] updating namespace I0508 18:16:20.030291 3653 namespace.go:141] [namespace-lister] updating namespace I0508 18:16:20.121646 3653 namespace.go:141] [release-service] updating namespace I0508 18:16:20.224801 3653 namespace.go:141] [segment-bridge] updating namespace W0508 18:16:21.102394 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0508 18:16:21.122445 3653 util.go:863] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0508 18:16:23.112783 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0508 18:16:23.134266 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0508 18:16:26.144863 3653 util.go:863] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0508 18:16:27.161303 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0508 18:16:27.167356 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0508 18:16:31.074089 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0508 18:16:31.074114 3653 util.go:863] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0508 18:16:31.191342 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0508 18:16:31.192860 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0508 18:16:31.219875 3653 util.go:863] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0508 18:16:31.219895 3653 util.go:863] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0508 18:16:31.248311 3653 util.go:863] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0508 18:16:31.248891 3653 util.go:863] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0508 18:16:43.275298 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-tpr9t I0508 18:16:43.275327 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-tpr9t I0508 18:16:43.478303 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressFirewall" totalItems=6 I0508 18:16:56.713948 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0508 18:17:20.391345 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressIP" totalItems=10 I0508 18:17:34.802656 3653 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0508 18:17:40.613334 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-66d9f6c79b-6qgm6 I0508 18:17:40.613361 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-66d9f6c79b-6qgm6 I0508 18:17:46.324520 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0508 18:17:46.688952 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0508 18:17:46.688984 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0508 18:17:57.853806 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0508 18:18:05.173941 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=78 I0508 18:18:15.173673 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=13 I0508 18:18:16.471195 3653 namespace.go:97] [managed-tenant-s4740] adding namespace I0508 18:18:16.472064 3653 namespace.go:101] [managed-tenant-s4740] adding namespace took 848.746µs I0508 18:18:16.499051 3653 namespace.go:141] [managed-tenant-s4740] updating namespace I0508 18:18:16.522089 3653 namespace.go:141] [managed-tenant-s4740] updating namespace I0508 18:18:17.576468 3653 namespace.go:141] [managed-tenant-s4740] updating namespace I0508 18:18:17.700405 3653 namespace.go:141] [managed-tenant-s4740] updating namespace I0508 18:18:17.761925 3653 namespace.go:141] [managed-tenant-s4740] updating namespace I0508 18:18:26.174514 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=81 I0508 18:18:34.590634 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressService" totalItems=10 I0508 18:18:40.486389 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressQoS" totalItems=8 I0508 18:19:44.172577 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=155 I0508 18:19:55.306227 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0508 18:20:13.810057 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.UserDefinedNetwork" totalItems=8 I0508 18:20:47.172956 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=625 I0508 18:22:51.175816 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=81 I0508 18:22:56.394475 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressIP" totalItems=6 I0508 18:23:05.178149 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=4 I0508 18:24:30.328404 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0508 18:25:05.806401 3653 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0508 18:25:29.048811 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-tp78n I0508 18:25:29.048848 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-tp78n I0508 18:25:29.489722 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressQoS" totalItems=8 I0508 18:25:33.041862 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-gsjbp I0508 18:25:33.041896 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-gsjbp I0508 18:25:36.033803 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-sdbdq I0508 18:25:36.033833 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-sdbdq I0508 18:25:37.857203 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0508 18:25:57.716929 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0508 18:26:31.482330 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressFirewall" totalItems=10 I0508 18:26:51.178531 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=45 I0508 18:27:01.175769 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=7 I0508 18:27:15.177653 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=10 I0508 18:27:45.310128 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0508 18:27:47.176161 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=14 I0508 18:28:31.595449 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressService" totalItems=10 I0508 18:28:55.815935 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.UserDefinedNetwork" totalItems=10 I0508 18:30:00.181356 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=8 I0508 18:30:05.677164 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 I0508 18:30:05.677198 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29637720-trd89 I0508 18:31:02.809833 3653 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 W0508 18:31:30.179148 3653 duration.go:461] Config duration recorder: measurement expired for pod/openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe678mp I0508 18:31:41.332379 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0508 18:32:26.485751 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressFirewall" totalItems=6 I0508 18:32:27.179389 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=10 I0508 18:32:35.398454 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressIP" totalItems=10 I0508 18:33:10.907652 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant/tsf-comp-zwgdh-on-pull-request-qsgzv-init-pod I0508 18:33:10.907683 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod default-tenant/tsf-comp-zwgdh-on-pull-request-qsgzv-init-pod I0508 18:33:20.181959 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=31 I0508 18:33:37.493549 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressQoS" totalItems=10 I0508 18:33:55.603436 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressService" totalItems=6 I0508 18:34:33.861290 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0508 18:34:41.179108 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=72 I0508 18:34:41.720793 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0508 18:34:42.179190 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=10 I0508 18:35:19.313226 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0508 18:35:29.181669 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=10 I0508 18:36:20.819627 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.UserDefinedNetwork" totalItems=8 I0508 18:37:37.182550 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=7 I0508 18:37:56.335619 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0508 18:38:42.186123 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=29 I0508 18:39:13.488987 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressFirewall" totalItems=8 I0508 18:39:23.858623 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dd5wn I0508 18:39:23.858653 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dd5wn I0508 18:39:27.184878 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service" totalItems=11 I0508 18:39:31.813340 3653 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0508 18:39:35.401938 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressIP" totalItems=7 I0508 18:39:42.864672 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0508 18:40:05.392455 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg I0508 18:40:05.392496 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29637730-gf8gg I0508 18:40:07.724980 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0508 18:41:12.182167 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=163 I0508 18:42:56.184885 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.NetworkPolicy" totalItems=8 I0508 18:43:14.497547 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressQoS" totalItems=11 I0508 18:43:41.607163 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressService" totalItems=10 I0508 18:43:49.823255 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.UserDefinedNetwork" totalItems=9 I0508 18:43:50.183185 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.EndpointSlice" totalItems=14 I0508 18:44:35.317490 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0508 18:44:40.493074 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressFirewall" totalItems=6 I0508 18:44:47.339435 3653 reflector.go:991] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0508 18:45:21.405605 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:141" type="*v1.EgressIP" totalItems=5 I0508 18:45:24.728777 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:141" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0508 18:46:00.185993 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace" totalItems=10 I0508 18:46:10.817525 3653 reflector.go:991] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0508 18:46:13.185537 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod" totalItems=133 I0508 18:47:34.868396 3653 reflector.go:991] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:141" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0508 18:47:49.872312 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod managed-tenant-s4740/affinity-assistant-20ab28ecc6-0 I0508 18:47:49.872341 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod managed-tenant-s4740/affinity-assistant-20ab28ecc6-0 I0508 18:47:49.969075 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *factory.egressIPPod managed-tenant-s4740/managed-kfc9d-verify-conforma-pod I0508 18:47:49.969104 3653 obj_retry.go:795] default/networkController: ignoring delete event for resource in terminal state *v1.Pod managed-tenant-s4740/managed-kfc9d-verify-conforma-pod I0508 18:47:54.189965 3653 reflector.go:991] "Watch close" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node" totalItems=70