+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-22.ec2.internal ++ [[ -n ip-10-0-2-22.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-22.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0511 17:57:23.176159097 - disable conntrack on geneve port' I0511 17:57:23.176159097 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0511 17:57:23.216918327 - starting ovnkube-node + echo 'I0511 17:57:23.216918327 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-22.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-22.ec2.internal --init-node ip-10-0-2-22.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-22.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0511 17:57:23.261229 3696 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0511 17:57:23.261285 3696 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-79a9d3cb87.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0511 17:57:23.262962 3696 kube.go:431] Waiting for certificate I0511 17:57:23.263000 3696 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0511 17:57:23.263034 3696 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0511 17:57:23.263914 3696 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0511 17:57:23.291749 3696 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0511 17:57:23.291770 3696 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0511 17:57:23.294452 3696 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0511 17:57:23.297049 3696 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rr6t4" I0511 17:57:23.308696 3696 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rr6t4" I0511 17:57:23.308789 3696 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0511 17:57:24.263446 3696 kube.go:438] Certificate found I0511 17:57:24.264100 3696 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0511 17:57:24.265249 3696 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0511 17:57:24.311390 3696 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-12 17:52:23 +0000 UTC" deadline="2026-05-12 14:12:22.178826114 +0000 UTC" I0511 17:57:24.311429 3696 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h14m57.867407669s" I0511 17:57:24.342049 3696 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0511 17:57:24.350450 3696 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0511 17:57:24.350519 3696 server.go:101] MetricServer registers OVS metrics I0511 17:57:24.350553 3696 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0511 17:57:24.350628 3696 factory.go:561] Starting watch factory I0511 17:57:24.350723 3696 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350745 3696 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350824 3696 server.go:105] MetricServer registers OVN DB metrics I0511 17:57:24.350841 3696 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350858 3696 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350860 3696 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350859 3696 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350872 3696 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350881 3696 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.350886 3696 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.351457 3696 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.351476 3696 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.351491 3696 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.354891 3696 controller_manager.go:370] Starting the ovnkube controller I0511 17:57:24.354905 3696 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-22.ec2.internal I0511 17:57:24.354959 3696 controller_manager.go:395] NBDB zone sync took: 45.965µs I0511 17:57:24.354970 3696 factory.go:561] Starting watch factory I0511 17:57:24.356008 3696 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0511 17:57:24.356028 3696 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0511 17:57:24.369015 3696 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0511 17:57:24.369040 3696 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0511 17:57:24.370961 3696 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.372704 3696 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0511 17:57:24.372735 3696 server.go:109] MetricServer registers OVN Controller metrics I0511 17:57:24.372704 3696 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.373058 3696 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.373190 3696 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.375876 3696 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.377818 3696 server.go:113] MetricServer registers OVN Northd metrics I0511 17:57:24.378942 3696 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0511 17:57:24.382431 3696 metrics.go:519] OVN Metrics Server starts to run ... I0511 17:57:24.456472 3696 factory.go:1977] *v1.Node informer cache synced successfully I0511 17:57:24.456510 3696 factory.go:1977] *v1.Service informer cache synced successfully I0511 17:57:24.456516 3696 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0511 17:57:24.456521 3696 factory.go:1977] *v1.Pod informer cache synced successfully I0511 17:57:24.456525 3696 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0511 17:57:24.456530 3696 factory.go:1977] *v1.Namespace informer cache synced successfully I0511 17:57:24.456620 3696 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0511 17:57:24.456634 3696 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0511 17:57:24.456654 3696 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0511 17:57:24.456642 3696 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0511 17:57:24.459701 3696 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0511 17:57:24.459713 3696 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0511 17:57:24.460449 3696 factory.go:1977] *v1.Pod informer cache synced successfully I0511 17:57:24.460459 3696 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0511 17:57:24.460463 3696 factory.go:1977] *v1.Namespace informer cache synced successfully I0511 17:57:24.460466 3696 factory.go:1977] *v1.Node informer cache synced successfully I0511 17:57:24.460469 3696 factory.go:1977] *v1.Service informer cache synced successfully I0511 17:57:24.460472 3696 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0511 17:57:24.461598 3696 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0511 17:57:24.461604 3696 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0511 17:57:24.461657 3696 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.461667 3696 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.464104 3696 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.466108 3696 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0511 17:57:24.466118 3696 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0511 17:57:24.467352 3696 factory.go:1977] *v1.EgressIP informer cache synced successfully I0511 17:57:24.467413 3696 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.467428 3696 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.469894 3696 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.471867 3696 factory.go:1977] *v1.EgressIP informer cache synced successfully I0511 17:57:24.473141 3696 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0511 17:57:24.473191 3696 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.473203 3696 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.475748 3696 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.477767 3696 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0511 17:57:24.479034 3696 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0511 17:57:24.479086 3696 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.479098 3696 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.481511 3696 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.483528 3696 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0511 17:57:24.485024 3696 factory.go:1977] *v1.EgressService informer cache synced successfully I0511 17:57:24.485096 3696 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.485110 3696 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.487589 3696 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.489517 3696 factory.go:1977] *v1.EgressService informer cache synced successfully I0511 17:57:24.490892 3696 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0511 17:57:24.491078 3696 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0511 17:57:24.491091 3696 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0511 17:57:24.493424 3696 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0511 17:57:24.495413 3696 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0511 17:57:24.496712 3696 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0511 17:57:24.496784 3696 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.496796 3696 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.496840 3696 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.496859 3696 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.499899 3696 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.499949 3696 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0511 17:57:24.500828 3696 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0511 17:57:24.501971 3696 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0511 17:57:24.501977 3696 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0511 17:57:24.501984 3696 factory.go:680] Watch Factory start up complete, took: 151.360367ms I0511 17:57:24.502054 3696 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0511 17:57:24.506386 3696 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0511 17:57:24.506397 3696 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0511 17:57:24.506404 3696 factory.go:680] Watch Factory start up complete, took: 151.433672ms I0511 17:57:24.506410 3696 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-22.ec2.internal" zone I0511 17:57:24.544039 3696 default_node_network_controller.go:710] Initializing the default node network controller I0511 17:57:24.610685 3696 udn_isolation.go:101] Starting UDN host isolation manager I0511 17:57:24.617192 3696 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0511 17:57:24.668986 3696 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0511 17:57:24.669084 3696 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0511 17:57:24.669102 3696 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0511 17:57:24.701957 3696 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0511 17:57:24.702022 3696 default_node_network_controller.go:809] Node ip-10-0-2-22.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0511 17:57:24.707665 3696 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.22"] k8s.ovn.org/zone-name:ip-10-0-2-22.ec2.internal] on node ip-10-0-2-22.ec2.internal I0511 17:57:24.732432 3696 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0511 17:57:24.739157 3696 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0511 17:57:24.739334 3696 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0511 17:57:24.739544 3696 gateway_init.go:263] Preparing Gateway I0511 17:57:24.739559 3696 gateway_shared_intf.go:1687] Creating new gateway I0511 17:57:24.756832 3696 controller_manager.go:429] Waiting for node in zone sync took: 250.407415ms I0511 17:57:24.760881 3696 controller_manager.go:611] Switching to transit router for layer2 networks I0511 17:57:24.760899 3696 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-22.ec2.internal I0511 17:57:24.774264 3696 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0511 17:57:24.780627 3696 controller_manager.go:290] SCTP support detected in OVN I0511 17:57:24.781278 3696 services_controller.go:69] Creating services controller for network=default I0511 17:57:24.781418 3696 udn_enabled_svc.go:80] Waiting for service informer to sync I0511 17:57:24.781559 3696 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0511 17:57:24.781610 3696 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0511 17:57:24.781633 3696 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0511 17:57:24.781647 3696 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0511 17:57:24.781888 3696 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0511 17:57:24.781941 3696 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0511 17:57:24.781970 3696 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 50.229µs I0511 17:57:24.781988 3696 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 35.501µs I0511 17:57:24.781998 3696 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 57.83µs I0511 17:57:24.782004 3696 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0511 17:57:24.782014 3696 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0511 17:57:24.782024 3696 default_network_controller.go:325] Starting the default network controller I0511 17:57:24.782038 3696 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 12.133µs I0511 17:57:24.782178 3696 udn_enabled_svc.go:95] Performing full resync I0511 17:57:24.782608 3696 udn_enabled_svc.go:99] Waiting for handler to sync I0511 17:57:24.782619 3696 udn_enabled_svc.go:104] Starting worker I0511 17:57:24.791692 3696 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0511 17:57:24.791751 3696 repair.go:33] Syncing exgw routes took 46.548µs I0511 17:57:24.791774 3696 default_network_controller.go:414] Starting all the Watchers... I0511 17:57:24.791916 3696 namespace.go:98] [openshift-console-user-settings] adding namespace I0511 17:57:24.791926 3696 namespace.go:98] [openshift-etcd] adding namespace I0511 17:57:24.791951 3696 namespace.go:98] [openshift-backplane-csm] adding namespace I0511 17:57:24.791962 3696 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0511 17:57:24.791982 3696 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0511 17:57:24.791990 3696 namespace.go:98] [open-cluster-management-2q7seai6iuch6pltn7smadh3stviopbe] adding namespace I0511 17:57:24.791989 3696 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0511 17:57:24.791993 3696 namespace.go:98] [openshift-cluster-version] adding namespace I0511 17:57:24.792005 3696 namespace.go:98] [openshift-kube-scheduler] adding namespace I0511 17:57:24.792006 3696 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0511 17:57:24.792005 3696 namespace.go:98] [openshift-route-controller-manager] adding namespace I0511 17:57:24.792013 3696 namespace.go:98] [openshift-monitoring] adding namespace I0511 17:57:24.792020 3696 namespace.go:98] [openshift-backplane-mobb] adding namespace I0511 17:57:24.792069 3696 namespace.go:98] [openshift-marketplace] adding namespace I0511 17:57:24.792112 3696 namespace.go:98] [kube-public] adding namespace I0511 17:57:24.792625 3696 namespace.go:102] [openshift-console-user-settings] adding namespace took 694.276µs I0511 17:57:24.792655 3696 namespace.go:98] [openshift-network-operator] adding namespace I0511 17:57:24.793115 3696 namespace.go:102] [openshift-network-operator] adding namespace took 454.041µs I0511 17:57:24.793131 3696 namespace.go:98] [openshift-service-ca-operator] adding namespace I0511 17:57:24.793639 3696 namespace.go:102] [openshift-service-ca-operator] adding namespace took 500.711µs I0511 17:57:24.793676 3696 namespace.go:98] [openshift-backplane-ai-agent] adding namespace I0511 17:57:24.794226 3696 namespace.go:102] [openshift-etcd] adding namespace took 2.288361ms I0511 17:57:24.794251 3696 namespace.go:98] [openshift-operators] adding namespace I0511 17:57:24.794725 3696 namespace.go:102] [openshift-backplane-csm] adding namespace took 2.7652ms I0511 17:57:24.794742 3696 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0511 17:57:24.795677 3696 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 3.700165ms I0511 17:57:24.795696 3696 namespace.go:98] [openshift-authentication] adding namespace I0511 17:57:24.796154 3696 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 4.156269ms I0511 17:57:24.796169 3696 namespace.go:98] [openshift-package-operator] adding namespace I0511 17:57:24.796642 3696 namespace.go:102] [open-cluster-management-2q7seai6iuch6pltn7smadh3stviopbe] adding namespace took 4.645585ms I0511 17:57:24.796654 3696 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0511 17:57:24.797082 3696 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.082626ms I0511 17:57:24.797090 3696 namespace.go:98] [openshift-host-network] adding namespace I0511 17:57:24.797525 3696 namespace.go:102] [openshift-cluster-version] adding namespace took 5.518254ms I0511 17:57:24.797546 3696 namespace.go:98] [kube-system] adding namespace I0511 17:57:24.798029 3696 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.016545ms I0511 17:57:24.798042 3696 namespace.go:98] [openshift-logging] adding namespace I0511 17:57:24.798477 3696 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.458945ms I0511 17:57:24.798490 3696 namespace.go:98] [openshift-multus] adding namespace I0511 17:57:24.798944 3696 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.924145ms I0511 17:57:24.798963 3696 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0511 17:57:24.799385 3696 namespace.go:102] [openshift-monitoring] adding namespace took 7.360906ms I0511 17:57:24.799397 3696 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0511 17:57:24.799857 3696 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.824451ms I0511 17:57:24.799874 3696 namespace.go:98] [default] adding namespace I0511 17:57:24.800322 3696 namespace.go:102] [openshift-marketplace] adding namespace took 8.237275ms I0511 17:57:24.800337 3696 namespace.go:98] [openshift-kube-apiserver] adding namespace I0511 17:57:24.800861 3696 namespace.go:102] [kube-public] adding namespace took 8.738614ms I0511 17:57:24.800876 3696 namespace.go:98] [openshift-image-registry] adding namespace I0511 17:57:24.801280 3696 namespace.go:102] [openshift-backplane-ai-agent] adding namespace took 7.598792ms I0511 17:57:24.801294 3696 namespace.go:98] [openshift-apiserver-operator] adding namespace I0511 17:57:24.801802 3696 namespace.go:102] [openshift-operators] adding namespace took 7.54416ms I0511 17:57:24.801816 3696 namespace.go:98] [openshift-config] adding namespace I0511 17:57:24.802253 3696 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.491269ms I0511 17:57:24.802267 3696 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0511 17:57:24.802703 3696 namespace.go:102] [openshift-authentication] adding namespace took 7.002505ms I0511 17:57:24.802716 3696 namespace.go:98] [openshift-infra] adding namespace I0511 17:57:24.803094 3696 namespace.go:102] [openshift-package-operator] adding namespace took 6.920799ms I0511 17:57:24.803105 3696 namespace.go:98] [openshift-network-console] adding namespace I0511 17:57:24.803528 3696 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.86821ms I0511 17:57:24.803550 3696 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace W0511 17:57:24.803948 3696 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-wwlkd: pod kube-system/global-pull-secret-syncer-wwlkd: no pod IPs found I0511 17:57:24.803995 3696 namespace.go:102] [openshift-host-network] adding namespace took 6.898961ms I0511 17:57:24.804008 3696 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0511 17:57:24.804455 3696 namespace.go:102] [kube-system] adding namespace took 6.904151ms I0511 17:57:24.804466 3696 namespace.go:98] [openshift-dns-operator] adding namespace W0511 17:57:24.804895 3696 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-hn9lj: pod openshift-multus/network-metrics-daemon-hn9lj: no pod IPs found I0511 17:57:24.804926 3696 namespace.go:102] [openshift-logging] adding namespace took 6.879837ms I0511 17:57:24.804936 3696 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0511 17:57:24.805388 3696 namespace.go:102] [openshift-multus] adding namespace took 6.894012ms I0511 17:57:24.805398 3696 namespace.go:98] [openshift-config-managed] adding namespace I0511 17:57:24.805808 3696 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.837994ms I0511 17:57:24.805823 3696 namespace.go:98] [openshift-backplane-srep] adding namespace I0511 17:57:24.806247 3696 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.845447ms I0511 17:57:24.806258 3696 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0511 17:57:24.806735 3696 namespace.go:102] [default] adding namespace took 6.855253ms I0511 17:57:24.806766 3696 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0511 17:57:24.807199 3696 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.856585ms I0511 17:57:24.807210 3696 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0511 17:57:24.807593 3696 namespace.go:102] [openshift-image-registry] adding namespace took 6.712523ms I0511 17:57:24.807605 3696 namespace.go:98] [openshift-operators-redhat] adding namespace I0511 17:57:24.807964 3696 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.6659ms I0511 17:57:24.808359 3696 namespace.go:102] [openshift-config] adding namespace took 6.538198ms I0511 17:57:24.808372 3696 namespace.go:98] [openshift] adding namespace I0511 17:57:24.808750 3696 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.478969ms I0511 17:57:24.808763 3696 namespace.go:98] [openshift-config-operator] adding namespace I0511 17:57:24.809207 3696 namespace.go:102] [openshift-infra] adding namespace took 6.486436ms I0511 17:57:24.809218 3696 namespace.go:98] [openshift-machine-config-operator] adding namespace I0511 17:57:24.809585 3696 namespace.go:102] [openshift-network-console] adding namespace took 6.475358ms I0511 17:57:24.809598 3696 namespace.go:98] [openshift-ingress-operator] adding namespace I0511 17:57:24.809997 3696 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.44125ms I0511 17:57:24.810015 3696 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0511 17:57:24.810621 3696 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.607308ms I0511 17:57:24.810636 3696 namespace.go:98] [dedicated-admin] adding namespace I0511 17:57:24.811086 3696 namespace.go:102] [openshift-dns-operator] adding namespace took 6.614684ms I0511 17:57:24.811099 3696 namespace.go:98] [openshift-controller-manager] adding namespace I0511 17:57:24.811469 3696 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.528202ms I0511 17:57:24.811482 3696 namespace.go:98] [openshift-apiserver] adding namespace I0511 17:57:24.811901 3696 namespace.go:102] [openshift-config-managed] adding namespace took 6.498956ms I0511 17:57:24.811913 3696 namespace.go:98] [openshift-backplane-cse] adding namespace I0511 17:57:24.812320 3696 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.491316ms I0511 17:57:24.812333 3696 namespace.go:98] [openshift-network-diagnostics] adding namespace I0511 17:57:24.812707 3696 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.444479ms I0511 17:57:24.812719 3696 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0511 17:57:24.813109 3696 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.336058ms I0511 17:57:24.813120 3696 namespace.go:98] [openshift-customer-monitoring] adding namespace I0511 17:57:24.813517 3696 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.302989ms I0511 17:57:24.813526 3696 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0511 17:57:24.813945 3696 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.33239ms I0511 17:57:24.813961 3696 namespace.go:98] [openshift-backplane-tam] adding namespace I0511 17:57:24.814337 3696 namespace.go:102] [openshift] adding namespace took 5.96099ms I0511 17:57:24.814351 3696 namespace.go:98] [openshift-backplane-cee] adding namespace I0511 17:57:24.814757 3696 namespace.go:102] [openshift-config-operator] adding namespace took 5.988756ms I0511 17:57:24.814772 3696 namespace.go:98] [openshift-console] adding namespace I0511 17:57:24.815081 3696 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.856986ms I0511 17:57:24.815101 3696 namespace.go:98] [openshift-node] adding namespace I0511 17:57:24.815539 3696 namespace.go:102] [openshift-ingress-operator] adding namespace took 5.935109ms I0511 17:57:24.815552 3696 namespace.go:98] [openshift-dns] adding namespace I0511 17:57:24.816110 3696 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.089148ms I0511 17:57:24.816127 3696 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0511 17:57:24.816523 3696 namespace.go:102] [dedicated-admin] adding namespace took 5.882449ms I0511 17:57:24.816537 3696 namespace.go:98] [openshift-console-operator] adding namespace I0511 17:57:24.817022 3696 namespace.go:102] [openshift-controller-manager] adding namespace took 5.917508ms I0511 17:57:24.817033 3696 namespace.go:98] [openshift-must-gather-operator] adding namespace I0511 17:57:24.817426 3696 namespace.go:102] [openshift-apiserver] adding namespace took 5.938859ms I0511 17:57:24.817441 3696 namespace.go:98] [openshift-network-node-identity] adding namespace W0511 17:57:24.817815 3696 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-wk5lb: pod openshift-network-diagnostics/network-check-target-wk5lb: no pod IPs found I0511 17:57:24.817865 3696 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.947273ms I0511 17:57:24.817876 3696 namespace.go:98] [openshift-backplane] adding namespace I0511 17:57:24.818293 3696 namespace.go:102] [openshift-network-diagnostics] adding namespace took 5.955706ms I0511 17:57:24.818305 3696 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0511 17:57:24.818728 3696 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.003914ms I0511 17:57:24.818741 3696 namespace.go:98] [kube-node-lease] adding namespace I0511 17:57:24.819114 3696 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.987969ms I0511 17:57:24.819130 3696 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0511 17:57:24.819474 3696 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.943743ms I0511 17:57:24.819485 3696 namespace.go:98] [openshift-ingress-canary] adding namespace I0511 17:57:24.819877 3696 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.909962ms I0511 17:57:24.819892 3696 namespace.go:98] [openshift-insights] adding namespace I0511 17:57:24.820242 3696 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.886342ms I0511 17:57:24.820253 3696 namespace.go:98] [openshift-authentication-operator] adding namespace I0511 17:57:24.821062 3696 namespace.go:102] [openshift-console] adding namespace took 6.286809ms I0511 17:57:24.821073 3696 namespace.go:98] [openshift-ingress] adding namespace I0511 17:57:24.821454 3696 namespace.go:102] [openshift-node] adding namespace took 6.346686ms I0511 17:57:24.821855 3696 namespace.go:102] [openshift-dns] adding namespace took 6.298ms I0511 17:57:24.822268 3696 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.134354ms I0511 17:57:24.822662 3696 namespace.go:102] [openshift-console-operator] adding namespace took 6.117722ms I0511 17:57:24.823038 3696 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.000477ms I0511 17:57:24.823050 3696 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0511 17:57:24.823406 3696 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.960497ms I0511 17:57:24.823815 3696 namespace.go:102] [openshift-backplane] adding namespace took 5.934788ms I0511 17:57:24.824205 3696 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.89578ms I0511 17:57:24.824590 3696 namespace.go:102] [kube-node-lease] adding namespace took 5.844662ms I0511 17:57:24.824977 3696 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.840889ms I0511 17:57:24.824991 3696 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0511 17:57:24.825398 3696 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.908435ms I0511 17:57:24.825411 3696 namespace.go:98] [openshift-machine-api] adding namespace I0511 17:57:24.825979 3696 namespace.go:102] [openshift-insights] adding namespace took 6.082025ms I0511 17:57:24.826557 3696 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.299556ms I0511 17:57:24.827088 3696 namespace.go:102] [openshift-ingress] adding namespace took 6.00971ms I0511 17:57:24.827676 3696 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 4.619511ms I0511 17:57:24.828280 3696 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 3.281709ms I0511 17:57:24.828882 3696 namespace.go:102] [openshift-machine-api] adding namespace took 3.464405ms W0511 17:57:24.828973 3696 master.go:458] Unable to parse local node ip-10-0-2-22.ec2.internal chassis-id annotation. Chassis may be removed during sync I0511 17:57:24.828982 3696 master.go:481] Removing stale chassis with ID/Name: 351e321c-9f8d-4c70-a6c8-eba0362d57fa, hostname: ip-10-0-2-22.ec2.internal I0511 17:57:24.829536 3696 master.go:548] Adding or Updating local node "ip-10-0-2-22.ec2.internal" for network "default" I0511 17:57:24.833155 3696 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-22.ec2.internal for the network default I0511 17:57:24.833459 3696 zone_ic_handler.go:195] Time taken to create transit switch: 290.392µs I0511 17:57:24.834243 3696 default_network_controller.go:823] Node add failed for ip-10-0-2-22.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal, error getting gateway config for node ip-10-0-2-22.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-22.ec2.internal", failed to update chassis to local for local node ip-10-0-2-22.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-22.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal] E0511 17:57:24.834318 3696 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-22.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal, error getting gateway config for node ip-10-0-2-22.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-22.ec2.internal", failed to update chassis to local for local node ip-10-0-2-22.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-22.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal] I0511 17:57:24.834378 3696 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0511 17:57:24.834417 3696 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0511 17:57:24.834435 3696 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0511 17:57:24.834404 3696 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-22.ec2.internal", UID:"24f41964-d563-4e89-94ea-f466e3e85ef4", APIVersion:"v1", ResourceVersion:"8293", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal, error getting gateway config for node ip-10-0-2-22.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-22.ec2.internal", failed to update chassis to local for local node ip-10-0-2-22.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-22.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal] I0511 17:57:24.834474 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0511 17:57:24.834529 3696 node_tracker.go:235] Node ip-10-0-2-22.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-22.ec2.internal" I0511 17:57:24.834546 3696 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0511 17:57:24.834560 3696 services_controller.go:574] Full service sync requested for network=default I0511 17:57:24.838956 3696 iptables.go:108] Creating table: filter chain: FORWARD I0511 17:57:24.844930 3696 iptables.go:108] Creating table: filter chain: OUTPUT I0511 17:57:24.850817 3696 gateway_shared_intf.go:1816] Gateway Creation Complete I0511 17:57:24.850835 3696 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-22.ec2.internal","mac-address":"0e:8c:d9:80:07:2d","ip-addresses":["10.0.2.22/24"],"ip-address":"10.0.2.22/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:351e321c-9f8d-4c70-a6c8-eba0362d57fa k8s.ovn.org/node-encap-ips:["10.0.2.22"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.22/24"} k8s.ovn.org/zone-name:ip-10-0-2-22.ec2.internal] on node ip-10-0-2-22.ec2.internal I0511 17:57:24.864669 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0511 17:57:24.864703 3696 master.go:548] Adding or Updating local node "ip-10-0-2-22.ec2.internal" for network "default" I0511 17:57:24.864743 3696 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0511 17:57:24.864757 3696 services_controller.go:574] Full service sync requested for network=default I0511 17:57:24.868522 3696 default_node_network_controller.go:519] Node connection status = connected W0511 17:57:24.869928 3696 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-wk5lb: pod openshift-network-diagnostics/network-check-target-wk5lb: no pod IPs found W0511 17:57:24.869952 3696 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-hn9lj: pod openshift-multus/network-metrics-daemon-hn9lj: no pod IPs found W0511 17:57:24.869959 3696 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-wwlkd: pod kube-system/global-pull-secret-syncer-wwlkd: no pod IPs found I0511 17:57:24.871086 3696 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-22.ec2.internal for the network default I0511 17:57:24.871317 3696 zone_ic_handler.go:195] Time taken to create transit switch: 218.444µs I0511 17:57:24.935351 3696 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0511 17:57:24.935373 3696 services_controller.go:208] Setting up event handlers for services for network=default I0511 17:57:24.935455 3696 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0511 17:57:24.935570 3696 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0511 17:57:24.935594 3696 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0511 17:57:25.036674 3696 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0511 17:57:25.036695 3696 repair.go:60] Starting repairing loop for services I0511 17:57:25.036797 3696 repair.go:131] Deleted 0 stale service LBs I0511 17:57:25.036816 3696 repair.go:137] Deleted 0 stale Chassis Template Vars I0511 17:57:25.036832 3696 repair.go:62] Finished repairing loop for services: 138.94µs I0511 17:57:25.036856 3696 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0511 17:57:25.036866 3696 services_controller.go:254] Starting workers for network=default I0511 17:57:25.037236 3696 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-hn9lj] creating logical port openshift-multus_network-metrics-daemon-hn9lj for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:25.037273 3696 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-wwlkd] creating logical port kube-system_global-pull-secret-syncer-wwlkd for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:25.037305 3696 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-wk5lb] creating logical port openshift-network-diagnostics_network-check-target-wk5lb for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:25.037425 3696 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-hn9lj I0511 17:57:25.037517 3696 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-wwlkd I0511 17:57:25.037517 3696 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-wk5lb I0511 17:57:25.062907 3696 pod.go:62] [kube-system/global-pull-secret-syncer-wwlkd] pod update took 25.560334ms I0511 17:57:25.062927 3696 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-wwlkd] addLogicalPort annotation time took 25.584439ms I0511 17:57:25.063841 3696 pods.go:271] [kube-system/global-pull-secret-syncer-wwlkd] addLogicalPort took 26.575416ms, libovsdb time 700.644µs I0511 17:57:25.066769 3696 pod.go:62] [openshift-network-diagnostics/network-check-target-wk5lb] pod update took 29.312843ms I0511 17:57:25.066781 3696 pod.go:62] [openshift-multus/network-metrics-daemon-hn9lj] pod update took 29.468026ms I0511 17:57:25.066785 3696 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-wk5lb] addLogicalPort annotation time took 29.349736ms I0511 17:57:25.066793 3696 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-hn9lj] addLogicalPort annotation time took 29.483285ms I0511 17:57:25.067656 3696 pods.go:271] [openshift-multus/network-metrics-daemon-hn9lj] addLogicalPort took 30.430379ms, libovsdb time 656.643µs I0511 17:57:25.068203 3696 pods.go:271] [openshift-network-diagnostics/network-check-target-wk5lb] addLogicalPort took 30.908331ms, libovsdb time 1.20763ms I0511 17:57:25.068515 3696 admin_network_policy_controller.go:227] Starting controller default-network-controller I0511 17:57:25.068538 3696 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0511 17:57:25.068551 3696 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0511 17:57:25.068555 3696 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0511 17:57:25.068734 3696 repair.go:29] Repairing admin network policies took 174.635µs I0511 17:57:25.068830 3696 repair.go:92] Repairing baseline admin network policies took 88.292µs I0511 17:57:25.069219 3696 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0511 17:57:25.069237 3696 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0511 17:57:25.071121 3696 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0511 17:57:25.071152 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0511 17:57:25.071166 3696 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0511 17:57:25.072958 3696 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0511 17:57:25.072980 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0511 17:57:25.072989 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0511 17:57:25.074864 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0511 17:57:25.074886 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0511 17:57:25.074893 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0511 17:57:25.076254 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0511 17:57:25.076273 3696 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0511 17:57:25.076282 3696 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0511 17:57:25.077785 3696 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0511 17:57:25.077802 3696 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0511 17:57:25.077808 3696 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0511 17:57:25.078093 3696 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0511 17:57:25.078111 3696 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0511 17:57:25.078118 3696 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0511 17:57:25.079312 3696 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0511 17:57:25.079329 3696 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0511 17:57:25.079337 3696 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0511 17:57:25.080404 3696 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0511 17:57:25.080423 3696 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0511 17:57:25.080434 3696 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0511 17:57:25.081609 3696 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0511 17:57:25.081627 3696 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0511 17:57:25.081637 3696 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0511 17:57:25.082709 3696 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0511 17:57:25.082729 3696 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0511 17:57:25.082739 3696 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0511 17:57:25.084356 3696 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0511 17:57:25.084373 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0511 17:57:25.084381 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0511 17:57:25.084857 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0511 17:57:25.084874 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0511 17:57:25.084882 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0511 17:57:25.086492 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0511 17:57:25.086529 3696 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0511 17:57:25.086536 3696 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0511 17:57:25.087101 3696 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0511 17:57:25.087117 3696 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0511 17:57:25.087126 3696 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0511 17:57:25.088196 3696 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0511 17:57:25.088214 3696 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0511 17:57:25.088225 3696 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0511 17:57:25.089560 3696 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0511 17:57:25.089580 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0511 17:57:25.089588 3696 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0511 17:57:25.090059 3696 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0511 17:57:25.090077 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0511 17:57:25.090084 3696 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0511 17:57:25.090599 3696 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0511 17:57:25.090615 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0511 17:57:25.090621 3696 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0511 17:57:25.090935 3696 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0511 17:57:25.090950 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0511 17:57:25.090957 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0511 17:57:25.091537 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0511 17:57:25.091554 3696 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0511 17:57:25.091563 3696 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0511 17:57:25.092652 3696 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0511 17:57:25.092671 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0511 17:57:25.092680 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0511 17:57:25.093161 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0511 17:57:25.093181 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0511 17:57:25.093190 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0511 17:57:25.093575 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0511 17:57:25.093592 3696 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0511 17:57:25.093601 3696 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0511 17:57:25.094554 3696 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0511 17:57:25.094572 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0511 17:57:25.094971 3696 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0511 17:57:25.094984 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0511 17:57:25.095661 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0511 17:57:25.095687 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0511 17:57:25.095695 3696 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0511 17:57:25.096156 3696 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0511 17:57:25.096175 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0511 17:57:25.096183 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0511 17:57:25.096722 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0511 17:57:25.096740 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0511 17:57:25.096760 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0511 17:57:25.097483 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0511 17:57:25.097518 3696 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0511 17:57:25.097529 3696 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0511 17:57:25.098045 3696 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0511 17:57:25.102271 3696 egressfirewall.go:374] Starting EgressFirewall controller I0511 17:57:25.102303 3696 controller.go:133] Adding controller egress-firewall-controller event handlers I0511 17:57:25.102310 3696 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0511 17:57:25.102351 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0511 17:57:25.102358 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0511 17:57:25.102365 3696 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0511 17:57:25.102370 3696 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0511 17:57:25.102675 3696 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0511 17:57:25.102690 3696 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0511 17:57:25.102697 3696 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0511 17:57:25.102707 3696 egressqos.go:193] Setting up event handlers for EgressQoS I0511 17:57:25.102720 3696 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0511 17:57:25.102926 3696 egressqos.go:245] Starting EgressQoS Controller I0511 17:57:25.102944 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0511 17:57:25.102954 3696 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0511 17:57:25.102990 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0511 17:57:25.102996 3696 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0511 17:57:25.103001 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0511 17:57:25.103007 3696 shared_informer.go:356] "Caches are synced" controller="egressqos" I0511 17:57:25.103013 3696 egressqos.go:259] Repairing EgressQoSes I0511 17:57:25.103018 3696 egressqos.go:399] Starting repairing loop for egressqos I0511 17:57:25.103111 3696 egressqos.go:401] Finished repairing loop for egressqos: 93.098µs I0511 17:57:25.103149 3696 egressservice_zone.go:132] Setting up event handlers for Egress Services I0511 17:57:25.103171 3696 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-22.ec2.internal I0511 17:57:25.103255 3696 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-22.ec2.internal : 85.195µs I0511 17:57:25.103418 3696 egressservice_zone.go:210] Starting Egress Services Controller I0511 17:57:25.103433 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0511 17:57:25.103443 3696 shared_informer.go:356] "Caches are synced" controller="egressservices" I0511 17:57:25.103452 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0511 17:57:25.103457 3696 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0511 17:57:25.103462 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0511 17:57:25.103466 3696 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0511 17:57:25.103471 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0511 17:57:25.103476 3696 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0511 17:57:25.103479 3696 egressservice_zone.go:228] Repairing Egress Services I0511 17:57:25.103748 3696 master_controller.go:88] Starting Admin Policy Based Route Controller I0511 17:57:25.103759 3696 external_controller.go:278] Starting Admin Policy Based Route Controller I0511 17:57:25.103829 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0511 17:57:25.103843 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 16.593µs I0511 17:57:25.103915 3696 default_network_controller.go:585] Completing all the Watchers took 312.129295ms I0511 17:57:25.103935 3696 default_network_controller.go:589] Starting unidling controllers I0511 17:57:25.103951 3696 unidle.go:46] Registering OVN SB ControllerEvent handler I0511 17:57:25.103962 3696 unidle.go:63] Populating Initial ContollerEvent events I0511 17:57:25.103977 3696 unidle.go:79] Setting up event handlers for services I0511 17:57:25.104102 3696 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0511 17:57:25.368876 3696 default_node_network_controller.go:519] Node connection status = connected I0511 17:57:25.634722 3696 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0511 17:57:25.869167 3696 default_node_network_controller.go:519] Node connection status = connected I0511 17:57:25.886515 3696 bridgeconfig.go:537] Gateway is ready I0511 17:57:25.886531 3696 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0511 17:57:25.898100 3696 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.22. Updating OVN encap IP. I0511 17:57:25.903240 3696 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.22 - it is already configured I0511 17:57:25.903253 3696 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.22/24:{} 172.20.0.1/32:{}]. Updating annotations. I0511 17:57:25.903485 3696 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.22/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-22.ec2.internal","mac-address":"0e:8c:d9:80:07:2d","ip-addresses":["10.0.2.22/24"],"ip-address":"10.0.2.22/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:351e321c-9f8d-4c70-a6c8-eba0362d57fa k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.22/24"}] on node ip-10-0-2-22.ec2.internal I0511 17:57:25.918620 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0511 17:57:25.918630 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0511 17:57:25.918642 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 24.729µs I0511 17:57:25.918644 3696 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0511 17:57:25.918660 3696 master.go:548] Adding or Updating local node "ip-10-0-2-22.ec2.internal" for network "default" I0511 17:57:25.918693 3696 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0511 17:57:25.918711 3696 services_controller.go:574] Full service sync requested for network=default I0511 17:57:25.918954 3696 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.22 I0511 17:57:25.919046 3696 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-22.ec2.internal I0511 17:57:25.922979 3696 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-22.ec2.internal. Updating no re-route policies I0511 17:57:25.923056 3696 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-22.ec2.internal I0511 17:57:25.923065 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-gd4v4 I0511 17:57:25.923070 3696 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-22.ec2.internal I0511 17:57:25.923076 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-gd4v4 I0511 17:57:25.923080 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-hn9lj I0511 17:57:25.923095 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-hn9lj I0511 17:57:25.923096 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-gd4v4 after 0 failed attempt(s) I0511 17:57:25.923097 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5jkpj I0511 17:57:25.923109 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-5fk7j I0511 17:57:25.923109 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-wk5lb I0511 17:57:25.923117 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5jkpj I0511 17:57:25.923125 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-wk5lb I0511 17:57:25.923125 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-6hzzs I0511 17:57:25.923137 3696 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-hn9lj] creating logical port openshift-multus_network-metrics-daemon-hn9lj for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:25.923141 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-9n5ls I0511 17:57:25.923151 3696 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-sxmfk I0511 17:57:25.923153 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-l4tkj I0511 17:57:25.923161 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-ps2m8 I0511 17:57:25.923146 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5jkpj after 0 failed attempt(s) I0511 17:57:25.923166 3696 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-sxmfk I0511 17:57:25.923168 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-l4tkj I0511 17:57:25.923171 3696 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-22.ec2.internal after 0 failed attempt(s) I0511 17:57:25.923058 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-tmlwp I0511 17:57:25.923178 3696 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-wwlkd I0511 17:57:25.923189 3696 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-sxmfk after 0 failed attempt(s) I0511 17:57:25.923191 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-l4tkj after 0 failed attempt(s) I0511 17:57:25.923163 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-9n5ls I0511 17:57:25.923201 3696 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-wwlkd I0511 17:57:25.923214 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-tmlwp I0511 17:57:25.923222 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-9n5ls after 0 failed attempt(s) I0511 17:57:25.923139 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-6hzzs I0511 17:57:25.923240 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-tmlwp after 0 failed attempt(s) I0511 17:57:25.923170 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-ps2m8 I0511 17:57:25.923246 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-22.ec2.internal I0511 17:57:25.923254 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-22.ec2.internal I0511 17:57:25.923255 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-ps2m8 after 0 failed attempt(s) I0511 17:57:25.923256 3696 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-wwlkd] creating logical port kube-system_global-pull-secret-syncer-wwlkd for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:25.923259 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-6hzzs after 0 failed attempt(s) I0511 17:57:25.923205 3696 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-wk5lb] creating logical port openshift-network-diagnostics_network-check-target-wk5lb for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:25.923268 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-22.ec2.internal after 0 failed attempt(s) I0511 17:57:25.923116 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-5fk7j I0511 17:57:25.923327 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-5fk7j after 0 failed attempt(s) I0511 17:57:25.923962 3696 pods.go:271] [kube-system/global-pull-secret-syncer-wwlkd] addLogicalPort took 713.668µs, libovsdb time 303.177µs I0511 17:57:25.923978 3696 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-wwlkd after 0 failed attempt(s) I0511 17:57:25.924091 3696 pods.go:271] [openshift-network-diagnostics/network-check-target-wk5lb] addLogicalPort took 944.78µs, libovsdb time 328.946µs I0511 17:57:25.924099 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-wk5lb after 0 failed attempt(s) I0511 17:57:25.924207 3696 pods.go:271] [openshift-multus/network-metrics-daemon-hn9lj] addLogicalPort took 1.082963ms, libovsdb time 529.318µs I0511 17:57:25.924213 3696 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-hn9lj after 0 failed attempt(s) I0511 17:57:25.934527 3696 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0511 17:57:25.946209 3696 iptables.go:108] Creating table: mangle chain: OUTPUT I0511 17:57:25.949405 3696 iptables.go:108] Creating table: nat chain: OUTPUT I0511 17:57:25.995056 3696 iptables.go:108] Creating table: nat chain: PREROUTING I0511 17:57:26.050912 3696 iptables.go:108] Creating table: filter chain: FORWARD I0511 17:57:26.067413 3696 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0511 17:57:26.067663 3696 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0511 17:57:26.067713 3696 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0511 17:57:26.067793 3696 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0511 17:57:26.067847 3696 gateway_init.go:325] Gateway and management port readiness took 1.20363296s I0511 17:57:26.067874 3696 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0511 17:57:26.067922 3696 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0511 17:57:26.067934 3696 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0511 17:57:26.067942 3696 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0511 17:57:26.068012 3696 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0511 17:57:26.068100 3696 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 59.074µs I0511 17:57:26.068430 3696 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0511 17:57:26.068457 3696 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.819µs I0511 17:57:26.068465 3696 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 35.143µs I0511 17:57:26.068471 3696 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0511 17:57:26.068479 3696 nad_controller.go:379] [node-nad-controller NAD controller]: started I0511 17:57:26.068484 3696 default_node_network_controller.go:899] Starting the default node network controller I0511 17:57:26.068489 3696 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0511 17:57:26.068535 3696 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 14.588µs I0511 17:57:26.068701 3696 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0511 17:57:26.068710 3696 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0511 17:57:26.068733 3696 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1f6a0 0xc0002dde30 0xc0002ddea0 0xc0002ddf10 0xc0002ee000 0xc0002ee070 0xc0002ee0e0 0xc000b2d8b0 0xc0002ee150 0xc0002ee1c0 0xc0003a42d0 0xc006177500 0xc0002ee310 map[0x2ff9080:0xc002313920 0x2ff95c0:0xc002313b60 0x2ffa2e0:0xc002313860 0x2ffa580:0xc0023138c0 0x2ffad60:0xc002313980 0x2ffb000:0xc0023139e0 0x2ffb2a0:0xc002313a40 0x2ffba80:0xc002313bc0 0x2ffbd20:0xc00231d860 0x3026980:0xc0017a03c0 0x3026d20:0xc0017a04e0 0x3027800:0xc000c2bb60 0x3029160:0xc00128e180 0x30298a0:0xc002313800 0x302a720:0xc0017a0360] 0xc000b22700 0} I0511 17:57:26.068794 3696 gateway.go:151] Starting gateway service sync I0511 17:57:26.069219 3696 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0511 17:57:26.071353 3696 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0511 17:57:26.094866 3696 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0511 17:57:26.096037 3696 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0511 17:57:26.097210 3696 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0511 17:57:26.130286 3696 gateway.go:168] Gateway service sync done. Time taken: 61.481563ms I0511 17:57:26.130609 3696 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31444 I0511 17:57:26.131058 3696 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1f6a0 0xc0002dde30 0xc0002ddea0 0xc0002ddf10 0xc0002ee000 0xc0002ee070 0xc0002ee0e0 0xc000b2d8b0 0xc0002ee150 0xc0002ee1c0 0xc0003a42d0 0xc006177500 0xc0002ee310 map[0x2ff9080:0xc002313920 0x2ff95c0:0xc002313b60 0x2ffa2e0:0xc002313860 0x2ffa580:0xc0023138c0 0x2ffad60:0xc002313980 0x2ffb000:0xc0023139e0 0x2ffb2a0:0xc002313a40 0x2ffba80:0xc002313bc0 0x2ffbd20:0xc00231d860 0x3026980:0xc0017a03c0 0x3026d20:0xc0017a04e0 0x3027800:0xc000c2bb60 0x3029160:0xc00128e180 0x30298a0:0xc002313800 0x302a720:0xc0017a0360] 0xc000b22700 0} I0511 17:57:26.131477 3696 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1f6a0 0xc0002dde30 0xc0002ddea0 0xc0002ddf10 0xc0002ee000 0xc0002ee070 0xc0002ee0e0 0xc000b2d8b0 0xc0002ee150 0xc0002ee1c0 0xc0003a42d0 0xc006177500 0xc0002ee310 map[0x2ff9080:0xc002313920 0x2ff95c0:0xc002313b60 0x2ffa2e0:0xc002313860 0x2ffa580:0xc0023138c0 0x2ffad60:0xc002313980 0x2ffb000:0xc0023139e0 0x2ffb2a0:0xc002313a40 0x2ffba80:0xc002313bc0 0x2ffbd20:0xc00231d860 0x3026980:0xc0017a03c0 0x3026d20:0xc0017a04e0 0x3027800:0xc000c2bb60 0x3029160:0xc00128e180 0x30298a0:0xc002313800 0x302a720:0xc0017a0360] 0xc000b22700 0} I0511 17:57:26.131535 3696 gateway.go:532] Reconciling gateway with updates I0511 17:57:26.137103 3696 gateway.go:339] Spawning Conntrack Rule Check Thread I0511 17:57:26.137188 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0511 17:57:26.137205 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0511 17:57:26.137224 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0511 17:57:26.137229 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0511 17:57:26.137242 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0511 17:57:26.137247 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0511 17:57:26.137259 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0511 17:57:26.137262 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0511 17:57:26.137276 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0511 17:57:26.137226 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0511 17:57:26.137284 3696 default_node_network_controller.go:944] Gateway and management port readiness took 68.56577ms I0511 17:57:26.137289 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0511 17:57:26.137293 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0511 17:57:26.137301 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0511 17:57:26.137306 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0511 17:57:26.137312 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0511 17:57:26.137315 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0511 17:57:26.137318 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0511 17:57:26.137315 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0511 17:57:26.137325 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0511 17:57:26.137329 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0511 17:57:26.137339 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0511 17:57:26.137348 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0511 17:57:26.137226 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0511 17:57:26.137350 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0511 17:57:26.137301 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0511 17:57:26.137205 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0511 17:57:26.137370 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0511 17:57:26.137372 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0511 17:57:26.137314 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0511 17:57:26.137383 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0511 17:57:26.137381 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0511 17:57:26.137388 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0511 17:57:26.137396 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0511 17:57:26.137329 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0511 17:57:26.137430 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0511 17:57:26.137293 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0511 17:57:26.137439 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0511 17:57:26.137446 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0511 17:57:26.137280 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0511 17:57:26.137455 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0511 17:57:26.137192 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0511 17:57:26.137463 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0511 17:57:26.137471 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0511 17:57:26.137472 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0511 17:57:26.137315 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0511 17:57:26.137479 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0511 17:57:26.137484 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0511 17:57:26.137485 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0511 17:57:26.137486 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0511 17:57:26.137490 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0511 17:57:26.137354 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0511 17:57:26.137513 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0511 17:57:26.137519 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0511 17:57:26.137522 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0511 17:57:26.137525 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0511 17:57:26.137472 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0511 17:57:26.137370 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0511 17:57:26.137543 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0511 17:57:26.137549 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0511 17:57:26.137470 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0511 17:57:26.137554 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0511 17:57:26.137558 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0511 17:57:26.137564 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0511 17:57:26.137482 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0511 17:57:26.137570 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0511 17:57:26.137577 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0511 17:57:26.137580 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0511 17:57:26.137211 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0511 17:57:26.137587 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0511 17:57:26.137591 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0511 17:57:26.137598 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0511 17:57:26.137368 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0511 17:57:26.137332 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0511 17:57:26.137605 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0511 17:57:26.137488 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0511 17:57:26.137610 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0511 17:57:26.137472 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0511 17:57:26.137355 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0511 17:57:26.137647 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0511 17:57:26.137654 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0511 17:57:26.137656 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0511 17:57:26.137659 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0511 17:57:26.137514 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0511 17:57:26.137672 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0511 17:57:26.137684 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0511 17:57:26.137662 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0511 17:57:26.137698 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0511 17:57:26.137700 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0511 17:57:26.137705 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0511 17:57:26.137706 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0511 17:57:26.137710 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0511 17:57:26.137516 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0511 17:57:26.137533 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0511 17:57:26.137354 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0511 17:57:26.137796 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0511 17:57:26.137605 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0511 17:57:26.137373 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0511 17:57:26.137608 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0511 17:57:26.137694 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0511 17:57:26.137883 3696 port_claim.go:71] Svc openshift-ingress/router-default: port 32632 is already open, no action needed I0511 17:57:26.137350 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0511 17:57:26.137901 3696 port_claim.go:71] Svc openshift-ingress/router-default: port 32334 is already open, no action needed I0511 17:57:26.137910 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0511 17:57:26.138292 3696 node_ip_handler_linux.go:160] Node IP manager is running W0511 17:57:26.174867 3696 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0511 17:57:26.175110 3696 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0511 17:57:26.175261 3696 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0511 17:57:26.182763 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0511 17:57:26.182797 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0511 17:57:26.182808 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0511 17:57:26.182816 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0511 17:57:26.182826 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0511 17:57:26.182835 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0511 17:57:26.182844 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0511 17:57:26.182852 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0511 17:57:26.182866 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0511 17:57:26.182877 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0511 17:57:26.182890 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0511 17:57:26.182898 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0511 17:57:26.182918 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0511 17:57:26.182928 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0511 17:57:26.182936 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0511 17:57:26.222864 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0511 17:57:26.222895 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0511 17:57:26.222902 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0511 17:57:26.222909 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0511 17:57:26.222916 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0511 17:57:26.222924 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0511 17:57:26.222932 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0511 17:57:26.222938 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0511 17:57:26.222963 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0511 17:57:26.222977 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0511 17:57:26.222986 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0511 17:57:26.222993 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0511 17:57:26.222999 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0511 17:57:26.223005 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0511 17:57:26.223010 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0511 17:57:26.261868 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0511 17:57:28.094053 3696 default_node_network_controller.go:1380] Starting node controller node sync W0511 17:57:28.094053 3696 egressip_healthcheck.go:76] Health checking using insecure connection I0511 17:57:28.094171 3696 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0511 17:57:28.105370 3696 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.293162ms I0511 17:57:28.105483 3696 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0511 17:57:28.105792 3696 egressservice_node.go:93] Setting up event handlers for Egress Services I0511 17:57:28.105896 3696 egressservice_node.go:183] Starting Egress Services Controller I0511 17:57:28.105915 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0511 17:57:28.105936 3696 shared_informer.go:356] "Caches are synced" controller="egressservices" I0511 17:57:28.105952 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0511 17:57:28.105966 3696 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0511 17:57:28.105976 3696 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0511 17:57:28.105988 3696 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0511 17:57:28.105995 3696 egressservice_node.go:197] Repairing Egress Services I0511 17:57:28.116090 3696 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0511 17:57:28.116103 3696 external_controller.go:278] Starting Admin Policy Based Route Controller I0511 17:57:28.116319 3696 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0511 17:57:28.116361 3696 link_network_manager.go:119] Link manager is running I0511 17:57:28.116394 3696 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0511 17:57:28.116567 3696 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0511 17:57:28.117034 3696 vrf_manager.go:128] VRF manager is running I0511 17:57:28.117301 3696 node_controller_manager.go:437] Removing flows to drop GARP I0511 17:57:28.117312 3696 gateway.go:532] Reconciling gateway with updates I0511 17:57:28.140350 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0511 17:57:28.140369 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0511 17:57:28.140374 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0511 17:57:28.140371 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0511 17:57:28.140385 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0511 17:57:28.140392 3696 port_claim.go:71] Svc openshift-ingress/router-default: port 32632 is already open, no action needed I0511 17:57:28.140399 3696 port_claim.go:71] Svc openshift-ingress/router-default: port 32334 is already open, no action needed I0511 17:57:28.140406 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0511 17:57:28.140420 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0511 17:57:28.140422 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0511 17:57:28.140355 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0511 17:57:28.140430 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0511 17:57:28.140447 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0511 17:57:28.140446 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0511 17:57:28.140448 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0511 17:57:28.140460 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0511 17:57:28.140374 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0511 17:57:28.140471 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0511 17:57:28.140471 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0511 17:57:28.140482 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0511 17:57:28.140490 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0511 17:57:28.140489 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0511 17:57:28.140516 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0511 17:57:28.140392 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0511 17:57:28.140522 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0511 17:57:28.140530 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0511 17:57:28.140433 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0511 17:57:28.140522 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0511 17:57:28.140545 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0511 17:57:28.140520 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0511 17:57:28.140391 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0511 17:57:28.140604 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0511 17:57:28.140606 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0511 17:57:28.140612 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0511 17:57:28.140612 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0511 17:57:28.140625 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0511 17:57:28.140628 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0511 17:57:28.140634 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0511 17:57:28.140633 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0511 17:57:28.140639 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0511 17:57:28.140643 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0511 17:57:28.140651 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0511 17:57:28.140386 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0511 17:57:28.140655 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0511 17:57:28.140659 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0511 17:57:28.140661 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0511 17:57:28.140662 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0511 17:57:28.140668 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0511 17:57:28.140669 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0511 17:57:28.140671 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0511 17:57:28.140675 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0511 17:57:28.140679 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0511 17:57:28.140600 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0511 17:57:28.140682 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0511 17:57:28.140684 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0511 17:57:28.140685 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0511 17:57:28.140682 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0511 17:57:28.140695 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0511 17:57:28.140703 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0511 17:57:28.140659 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0511 17:57:28.140728 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0511 17:57:28.140685 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0511 17:57:28.140671 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0511 17:57:28.140778 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0511 17:57:28.140635 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0511 17:57:28.140652 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0511 17:57:28.140492 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0511 17:57:28.140835 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0511 17:57:28.140408 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0511 17:57:28.140845 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0511 17:57:28.140619 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0511 17:57:28.140873 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0511 17:57:28.140484 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0511 17:57:28.140472 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0511 17:57:28.140474 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0511 17:57:28.140435 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0511 17:57:28.140516 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0511 17:57:28.140936 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0511 17:57:28.140941 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0511 17:57:28.140520 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0511 17:57:28.140532 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0511 17:57:28.140976 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0511 17:57:28.140534 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0511 17:57:28.140635 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0511 17:57:28.140642 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0511 17:57:28.141059 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0511 17:57:28.140648 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0511 17:57:28.140653 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0511 17:57:28.141098 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0511 17:57:28.140664 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0511 17:57:28.140673 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0511 17:57:28.140668 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0511 17:57:28.141180 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0511 17:57:28.140679 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0511 17:57:28.141202 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0511 17:57:28.140684 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0511 17:57:28.141226 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0511 17:57:28.140689 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0511 17:57:28.140686 3696 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0511 17:57:28.141292 3696 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0511 17:57:28.141298 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0511 17:57:28.179784 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0511 17:57:28.179809 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0511 17:57:28.179816 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0511 17:57:28.179822 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0511 17:57:28.179827 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0511 17:57:28.218211 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0511 17:57:28.218225 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218215 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0511 17:57:28.218248 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0511 17:57:28.218254 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218278 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0511 17:57:28.218297 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0511 17:57:28.218315 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0511 17:57:28.218322 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0511 17:57:28.218329 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0511 17:57:28.218335 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0511 17:57:28.218342 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0511 17:57:28.218348 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0511 17:57:28.218354 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218360 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0511 17:57:28.218365 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218370 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0511 17:57:28.218376 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0511 17:57:28.218381 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0511 17:57:28.218387 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218391 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218398 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218402 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0511 17:57:28.218407 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0511 17:57:28.257150 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0511 17:57:28.257175 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0511 17:57:28.257183 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0511 17:57:28.257190 3696 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0511 17:57:29.668251 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0511 17:57:29.668284 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 53.204µs I0511 17:57:29.696247 3696 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-58f8556b85-pnznl] creating logical port openshift-image-registry_image-registry-58f8556b85-pnznl for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.696379 3696 kube.go:256] Updating pod openshift-image-registry/image-registry-58f8556b85-pnznl I0511 17:57:29.723810 3696 pod.go:62] [openshift-image-registry/image-registry-58f8556b85-pnznl] pod update took 27.46112ms I0511 17:57:29.723831 3696 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-58f8556b85-pnznl] addLogicalPort annotation time took 27.488284ms I0511 17:57:29.725096 3696 pods.go:271] [openshift-image-registry/image-registry-58f8556b85-pnznl] addLogicalPort took 28.875097ms, libovsdb time 967.049µs I0511 17:57:29.738832 3696 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-khg57] creating logical port openshift-dns_dns-default-khg57 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.738941 3696 kube.go:256] Updating pod openshift-dns/dns-default-khg57 I0511 17:57:29.744815 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-gwxgf for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.744921 3696 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf I0511 17:57:29.744922 3696 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-g4qdx] creating logical port openshift-console-operator_console-operator-66dbbb6468-g4qdx for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.745104 3696 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-g4qdx I0511 17:57:29.750604 3696 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-lglzb for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.750720 3696 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb I0511 17:57:29.750881 3696 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-77758cd455-8z89h] creating logical port openshift-ingress_router-default-77758cd455-8z89h for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.750980 3696 kube.go:256] Updating pod openshift-ingress/router-default-77758cd455-8z89h I0511 17:57:29.750985 3696 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-ptg2w for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.751075 3696 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w I0511 17:57:29.752918 3696 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-t2jfz for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.753060 3696 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz I0511 17:57:29.753079 3696 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5f7f58d6b5-kmnkj] creating logical port openshift-ingress_router-default-5f7f58d6b5-kmnkj for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.753168 3696 kube.go:256] Updating pod openshift-ingress/router-default-5f7f58d6b5-kmnkj I0511 17:57:29.762397 3696 pod.go:62] [openshift-dns/dns-default-khg57] pod update took 23.487165ms I0511 17:57:29.762418 3696 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-khg57] addLogicalPort annotation time took 23.510169ms I0511 17:57:29.763550 3696 pods.go:271] [openshift-dns/dns-default-khg57] addLogicalPort took 24.735272ms, libovsdb time 833.397µs I0511 17:57:29.763872 3696 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-89qhz for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.763918 3696 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz I0511 17:57:29.765395 3696 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf] pod update took 20.497416ms I0511 17:57:29.765410 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf] addLogicalPort annotation time took 20.515581ms I0511 17:57:29.766407 3696 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf] addLogicalPort took 21.605606ms, libovsdb time 789.055µs I0511 17:57:29.769865 3696 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-g4qdx] pod update took 24.783658ms I0511 17:57:29.769885 3696 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-g4qdx] addLogicalPort annotation time took 24.807076ms I0511 17:57:29.770510 3696 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb] pod update took 19.801018ms I0511 17:57:29.770529 3696 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb] addLogicalPort annotation time took 19.837159ms I0511 17:57:29.770816 3696 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-g4qdx] addLogicalPort took 25.904786ms, libovsdb time 702.069µs I0511 17:57:29.771647 3696 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb] addLogicalPort took 21.054585ms, libovsdb time 723.621µs I0511 17:57:29.775362 3696 pod.go:62] [openshift-ingress/router-default-77758cd455-8z89h] pod update took 24.404415ms I0511 17:57:29.775384 3696 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-77758cd455-8z89h] addLogicalPort annotation time took 24.428958ms I0511 17:57:29.775439 3696 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w] pod update took 24.383068ms I0511 17:57:29.775461 3696 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w] addLogicalPort annotation time took 24.406885ms I0511 17:57:29.777524 3696 pods.go:271] [openshift-ingress/router-default-77758cd455-8z89h] addLogicalPort took 26.654302ms, libovsdb time 1.797535ms I0511 17:57:29.777727 3696 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-sg7fv] creating logical port openshift-insights_insights-operator-6dbb575d6c-sg7fv for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.777891 3696 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-sg7fv I0511 17:57:29.778445 3696 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w] addLogicalPort took 27.467806ms, libovsdb time 1.282754ms I0511 17:57:29.778845 3696 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-vwvnv for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.778998 3696 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv I0511 17:57:29.782830 3696 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-4c49g for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.783078 3696 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g I0511 17:57:29.785128 3696 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz] pod update took 32.090197ms I0511 17:57:29.785197 3696 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz] addLogicalPort annotation time took 32.160574ms I0511 17:57:29.785753 3696 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-vjspg for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.786030 3696 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg I0511 17:57:29.786591 3696 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-kvhn4] creating logical port openshift-ingress-canary_ingress-canary-kvhn4 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.786689 3696 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-kvhn4 I0511 17:57:29.786846 3696 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz] addLogicalPort took 33.936666ms, libovsdb time 1.080724ms I0511 17:57:29.788443 3696 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz] pod update took 24.532708ms I0511 17:57:29.788532 3696 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz] addLogicalPort annotation time took 24.622253ms I0511 17:57:29.788896 3696 pod.go:62] [openshift-ingress/router-default-5f7f58d6b5-kmnkj] pod update took 35.747913ms I0511 17:57:29.788929 3696 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5f7f58d6b5-kmnkj] addLogicalPort annotation time took 35.789576ms I0511 17:57:29.791178 3696 pods.go:271] [openshift-ingress/router-default-5f7f58d6b5-kmnkj] addLogicalPort took 38.103926ms, libovsdb time 1.050809ms I0511 17:57:29.791269 3696 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz] addLogicalPort took 27.404301ms, libovsdb time 1.623873ms I0511 17:57:29.796942 3696 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-sg7fv] pod update took 19.119292ms I0511 17:57:29.796966 3696 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-sg7fv] addLogicalPort annotation time took 19.145525ms I0511 17:57:29.797921 3696 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-sg7fv] addLogicalPort took 20.217428ms, libovsdb time 671.416µs I0511 17:57:29.798933 3696 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv] pod update took 19.979622ms I0511 17:57:29.798951 3696 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv] addLogicalPort annotation time took 20.001891ms I0511 17:57:29.800201 3696 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv] addLogicalPort took 21.364432ms, libovsdb time 967.347µs I0511 17:57:29.800258 3696 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-wmn76 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:29.800323 3696 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76 I0511 17:57:29.802656 3696 pod.go:62] [openshift-ingress-canary/ingress-canary-kvhn4] pod update took 16.007462ms I0511 17:57:29.802672 3696 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-kvhn4] addLogicalPort annotation time took 16.026713ms I0511 17:57:29.803941 3696 pods.go:271] [openshift-ingress-canary/ingress-canary-kvhn4] addLogicalPort took 17.357722ms, libovsdb time 900.572µs I0511 17:57:29.807293 3696 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg] pod update took 21.354721ms I0511 17:57:29.807310 3696 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg] addLogicalPort annotation time took 21.373698ms I0511 17:57:29.807573 3696 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g] pod update took 24.627188ms I0511 17:57:29.807594 3696 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g] addLogicalPort annotation time took 24.677294ms I0511 17:57:29.808517 3696 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg] addLogicalPort took 22.773277ms, libovsdb time 817.737µs I0511 17:57:29.808925 3696 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g] addLogicalPort took 26.112618ms, libovsdb time 1.034661ms I0511 17:57:29.813557 3696 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76] pod update took 13.253331ms I0511 17:57:29.813574 3696 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76] addLogicalPort annotation time took 13.274158ms I0511 17:57:29.814763 3696 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76] addLogicalPort took 14.515682ms, libovsdb time 862.561µs I0511 17:57:30.191036 3696 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-g4qdx e2d5512f0254c99fb18e7b442590145e7d1ac933ac6850db61cb842d722103f0 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-g4qdx e2d5512f0254c99fb18e7b442590145e7d1ac933ac6850db61cb842d722103f0 network default NAD default NAD key ] I0511 17:57:30.195045 3696 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz 5624971e9fcec99b3c677f850c5968f4b77e35b6bce904c9396d47b658e6345c network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz 5624971e9fcec99b3c677f850c5968f4b77e35b6bce904c9396d47b658e6345c network default NAD default NAD key ] I0511 17:57:30.222601 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-89qhz, hostIfaceName: 5624971e9fcec99, network: default, NAD default, SandboxID: "5624971e9fcec99b3c677f850c5968f4b77e35b6bce904c9396d47b658e6345c", PCI device ID: , UID: "2ee22cb9-3c22-448c-8301-2f7f13626111", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0511 17:57:30.223055 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-g4qdx, hostIfaceName: e2d5512f0254c99, network: default, NAD default, SandboxID: "e2d5512f0254c99fb18e7b442590145e7d1ac933ac6850db61cb842d722103f0", PCI device ID: , UID: "e972b7ad-cdd7-49b3-b093-333bfc0711ef", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0511 17:57:30.235233 3696 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb ac2b05de11e554f8207821f87fdf2a999d07cf4ca0c3dcec26a96a3349b07626 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb ac2b05de11e554f8207821f87fdf2a999d07cf4ca0c3dcec26a96a3349b07626 network default NAD default NAD key ] I0511 17:57:30.241142 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-174.ec2.internal I0511 17:57:30.241163 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-174.ec2.internal: 25.482µs E0511 17:57:30.241303 3696 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-174.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:57:30.241324 3696 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-174.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-174.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-174.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-174.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-174.ec2.internal, error parsing annotation for node ip-10-0-0-174.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] I0511 17:57:30.241633 3696 egressfirewall.go:723] Syncing node "ip-10-0-0-174.ec2.internal" for egress firewall I0511 17:57:30.241743 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-174.ec2.internal in network "default" I0511 17:57:30.241763 3696 node_tracker.go:220] Node ip-10-0-0-174.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:57:30.247273 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-lglzb, hostIfaceName: ac2b05de11e554f, network: default, NAD default, SandboxID: "ac2b05de11e554f8207821f87fdf2a999d07cf4ca0c3dcec26a96a3349b07626", PCI device ID: , UID: "48a4b650-0035-41ce-9f23-620f92bf25ea", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] E0511 17:57:30.254189 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-174.ec2.internal annotation: unexpected end of JSON input I0511 17:57:30.254310 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0511 17:57:30.254348 3696 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-174.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:57:30.277153 3696 egressfirewall.go:723] Syncing node "ip-10-0-0-174.ec2.internal" for egress firewall I0511 17:57:30.277181 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0511 17:57:30.277221 3696 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-174.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:57:30.277241 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-174.ec2.internal E0511 17:57:30.277247 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-174.ec2.internal annotation: unexpected end of JSON input I0511 17:57:30.277255 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-174.ec2.internal: 15.128µs I0511 17:57:30.280212 3696 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-g4qdx e2d5512f0254c99fb18e7b442590145e7d1ac933ac6850db61cb842d722103f0 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-g4qdx e2d5512f0254c99fb18e7b442590145e7d1ac933ac6850db61cb842d722103f0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:9f:50:84:08:ab\",\"name\":\"e2d5512f0254c99\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/63a3fc99-3a27-4c40-90c4-3ba857182167\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.284011 3696 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz 67c9b01eb28e5980da1a11b4f0c0202c6fa6b16243a96a850e2072b56d56b716 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz 67c9b01eb28e5980da1a11b4f0c0202c6fa6b16243a96a850e2072b56d56b716 network default NAD default NAD key ] I0511 17:57:30.284871 3696 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz 5624971e9fcec99b3c677f850c5968f4b77e35b6bce904c9396d47b658e6345c network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-89qhz 5624971e9fcec99b3c677f850c5968f4b77e35b6bce904c9396d47b658e6345c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:3e:05:56:3c:de\",\"name\":\"5624971e9fcec99\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1453c08-e158-47bb-b65b-35844239eb99\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.293284 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-t2jfz, hostIfaceName: 67c9b01eb28e598, network: default, NAD default, SandboxID: "67c9b01eb28e5980da1a11b4f0c0202c6fa6b16243a96a850e2072b56d56b716", PCI device ID: , UID: "18c65a60-c4dd-4cd6-8014-2e9b35d3db4c", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0511 17:57:30.299473 3696 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb ac2b05de11e554f8207821f87fdf2a999d07cf4ca0c3dcec26a96a3349b07626 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lglzb ac2b05de11e554f8207821f87fdf2a999d07cf4ca0c3dcec26a96a3349b07626 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:3f:57:d6:4c:09\",\"name\":\"ac2b05de11e554f\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b3dc50e-8532-4429-ad04-bc07ffcf1d49\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.342845 3696 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz 67c9b01eb28e5980da1a11b4f0c0202c6fa6b16243a96a850e2072b56d56b716 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-t2jfz 67c9b01eb28e5980da1a11b4f0c0202c6fa6b16243a96a850e2072b56d56b716 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:91:53:db:49:ec\",\"name\":\"67c9b01eb28e598\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cc5df70-495d-4899-b028-9c04ffd6475f\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err E0511 17:57:30.355018 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-174.ec2.internal annotation: unexpected end of JSON input E0511 17:57:30.355058 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-174.ec2.internal annotation: unexpected end of JSON input I0511 17:57:30.355085 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0511 17:57:30.355142 3696 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-174.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-174.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-174.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-174.ec2.internal" I0511 17:57:30.363390 3696 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-sg7fv e444a348846d43a1095b68bbf1be7101828b8b3735f62c58eb9f07f1221066d4 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-sg7fv e444a348846d43a1095b68bbf1be7101828b8b3735f62c58eb9f07f1221066d4 network default NAD default NAD key ] I0511 17:57:30.371289 3696 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76 e6d081108b818430c9115e8c3235949571f14602f2f9ecf6b34aa09ee83f213b network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76 e6d081108b818430c9115e8c3235949571f14602f2f9ecf6b34aa09ee83f213b network default NAD default NAD key ] I0511 17:57:30.373455 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0511 17:57:30.375355 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-sg7fv, hostIfaceName: e444a348846d43a, network: default, NAD default, SandboxID: "e444a348846d43a1095b68bbf1be7101828b8b3735f62c58eb9f07f1221066d4", PCI device ID: , UID: "826fae3c-d3ff-49f9-8476-53e0d017010b", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0511 17:57:30.385587 3696 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg 8ba10051633740999e320af7f2a99945bd0b6f40a84d87334a6c76fe3efd20b7 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg 8ba10051633740999e320af7f2a99945bd0b6f40a84d87334a6c76fe3efd20b7 network default NAD default NAD key ] I0511 17:57:30.389677 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-wmn76, hostIfaceName: e6d081108b81843, network: default, NAD default, SandboxID: "e6d081108b818430c9115e8c3235949571f14602f2f9ecf6b34aa09ee83f213b", PCI device ID: , UID: "2894edff-c2dc-4f87-bafe-3d9a5eb123df", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0511 17:57:30.400340 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-vjspg, hostIfaceName: 8ba100516337409, network: default, NAD default, SandboxID: "8ba10051633740999e320af7f2a99945bd0b6f40a84d87334a6c76fe3efd20b7", PCI device ID: , UID: "fbd96772-653b-40d1-bc59-7714a9d2c8a0", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0511 17:57:30.417685 3696 cni.go:452] [openshift-ingress-canary/ingress-canary-kvhn4 fcbd82b2d0d524a376901a48faeccb7a730f0957654f3a4650ae4bf6ff7bdb0c network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-kvhn4 fcbd82b2d0d524a376901a48faeccb7a730f0957654f3a4650ae4bf6ff7bdb0c network default NAD default NAD key ] I0511 17:57:30.427156 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-kvhn4, hostIfaceName: fcbd82b2d0d524a, network: default, NAD default, SandboxID: "fcbd82b2d0d524a376901a48faeccb7a730f0957654f3a4650ae4bf6ff7bdb0c", PCI device ID: , UID: "145bb98e-3d71-4b80-91de-af205daa18e9", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0511 17:57:30.432541 3696 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-sg7fv e444a348846d43a1095b68bbf1be7101828b8b3735f62c58eb9f07f1221066d4 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-sg7fv e444a348846d43a1095b68bbf1be7101828b8b3735f62c58eb9f07f1221066d4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:a0:ee:1c:60:54\",\"name\":\"e444a348846d43a\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e94e0d4-835a-427d-b206-2b9a2ac87b05\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.444360 3696 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76 e6d081108b818430c9115e8c3235949571f14602f2f9ecf6b34aa09ee83f213b network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-wmn76 e6d081108b818430c9115e8c3235949571f14602f2f9ecf6b34aa09ee83f213b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:7b:c2:b0:0c:2b\",\"name\":\"e6d081108b81843\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7500ad01-d215-4d2e-866e-931d22cb5166\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.457231 3696 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg 8ba10051633740999e320af7f2a99945bd0b6f40a84d87334a6c76fe3efd20b7 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vjspg 8ba10051633740999e320af7f2a99945bd0b6f40a84d87334a6c76fe3efd20b7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:12:30:cf:f1:a7\",\"name\":\"8ba100516337409\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/902c3800-f419-41bb-9866-10b523206135\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.477875 3696 cni.go:473] [openshift-ingress-canary/ingress-canary-kvhn4 fcbd82b2d0d524a376901a48faeccb7a730f0957654f3a4650ae4bf6ff7bdb0c network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-kvhn4 fcbd82b2d0d524a376901a48faeccb7a730f0957654f3a4650ae4bf6ff7bdb0c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:0f:de:16:2d:db\",\"name\":\"fcbd82b2d0d524a\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/03f99df4-d6bd-4997-89c4-bfabfd7a08ee\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:30.480492 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-174.ec2.internal I0511 17:57:30.480594 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-174.ec2.internal: 106.135µs I0511 17:57:30.480602 3696 egressfirewall.go:723] Syncing node "ip-10-0-0-174.ec2.internal" for egress firewall I0511 17:57:30.480525 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:57:30.501570 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:57:30.722466 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:57:32.030641 3696 cni.go:452] [openshift-network-diagnostics/network-check-target-wk5lb 3c521409403f01332c7e76267cce61cfaad1ab28e50ee3654381c2096f11d323 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-wk5lb 3c521409403f01332c7e76267cce61cfaad1ab28e50ee3654381c2096f11d323 network default NAD default NAD key ] I0511 17:57:32.031995 3696 cni.go:452] [kube-system/global-pull-secret-syncer-wwlkd 596d78931a358d120b1a7051a9754899450e2b9b77bdce50345de739e0da7b97 network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-wwlkd 596d78931a358d120b1a7051a9754899450e2b9b77bdce50345de739e0da7b97 network default NAD default NAD key ] I0511 17:57:32.047151 3696 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-wwlkd, hostIfaceName: 596d78931a358d1, network: default, NAD default, SandboxID: "596d78931a358d120b1a7051a9754899450e2b9b77bdce50345de739e0da7b97", PCI device ID: , UID: "ef812d1f-7776-4f0f-8494-fb1a03e267ba", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0511 17:57:32.047350 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-wk5lb, hostIfaceName: 3c521409403f013, network: default, NAD default, SandboxID: "3c521409403f01332c7e76267cce61cfaad1ab28e50ee3654381c2096f11d323", PCI device ID: , UID: "9ebc119f-572c-4647-aafa-a5ad6517d878", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0511 17:57:32.104197 3696 cni.go:473] [kube-system/global-pull-secret-syncer-wwlkd 596d78931a358d120b1a7051a9754899450e2b9b77bdce50345de739e0da7b97 network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-wwlkd 596d78931a358d120b1a7051a9754899450e2b9b77bdce50345de739e0da7b97 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:e8:e0:bc:74:a5\",\"name\":\"596d78931a358d1\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41e94d15-0c8a-4c75-a305-436f81b48a4b\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:32.112734 3696 cni.go:473] [openshift-network-diagnostics/network-check-target-wk5lb 3c521409403f01332c7e76267cce61cfaad1ab28e50ee3654381c2096f11d323 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-wk5lb 3c521409403f01332c7e76267cce61cfaad1ab28e50ee3654381c2096f11d323 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:46:39:b0:be:3f\",\"name\":\"3c521409403f013\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2273010-5f04-4ed3-8cdc-4e7452138198\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:34.827041 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:57:40.597867 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:57:41.161085 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-174.ec2.internal I0511 17:57:41.161102 3696 egressfirewall.go:723] Syncing node "ip-10-0-0-174.ec2.internal" for egress firewall I0511 17:57:41.161106 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-174.ec2.internal: 27.789µs I0511 17:57:41.161162 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:57:43.166124 3696 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0511 17:57:43.212578 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0511 17:57:43.234371 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0511 17:57:44.365386 3696 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0511 17:57:44.366437 3696 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.023452ms I0511 17:57:44.397917 3696 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0511 17:57:44.499947 3696 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-h67g8 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:44.500056 3696 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8 I0511 17:57:44.516709 3696 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8] pod update took 16.680244ms I0511 17:57:44.516739 3696 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8] addLogicalPort annotation time took 16.713603ms I0511 17:57:44.518237 3696 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8] addLogicalPort took 18.310486ms, libovsdb time 1.029565ms I0511 17:57:44.858057 3696 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8 e2cfbe5a2976874197a6cd67b35a9dcbb651d53927be519907472902ca63f10f network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8 e2cfbe5a2976874197a6cd67b35a9dcbb651d53927be519907472902ca63f10f network default NAD default NAD key ] I0511 17:57:44.867394 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-h67g8, hostIfaceName: e2cfbe5a2976874, network: default, NAD default, SandboxID: "e2cfbe5a2976874197a6cd67b35a9dcbb651d53927be519907472902ca63f10f", PCI device ID: , UID: "667c8163-38b8-46c9-b9ee-1ddd21fd7274", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0511 17:57:44.912808 3696 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8 e2cfbe5a2976874197a6cd67b35a9dcbb651d53927be519907472902ca63f10f network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-h67g8 e2cfbe5a2976874197a6cd67b35a9dcbb651d53927be519907472902ca63f10f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:86:0b:ba:fa:c7\",\"name\":\"e2cfbe5a2976874\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/73ea0e92-02f7-4aad-962b-c31f3c931f28\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:57:45.009994 3696 namespace.go:98] [openshift-service-ca] adding namespace I0511 17:57:45.010823 3696 namespace.go:102] [openshift-service-ca] adding namespace took 810.079µs I0511 17:57:45.027849 3696 namespace.go:142] [openshift-service-ca] updating namespace I0511 17:57:46.716958 3696 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-wv6mq] creating logical port openshift-service-ca_service-ca-d9cb48497-wv6mq for pod on switch ip-10-0-2-22.ec2.internal I0511 17:57:46.717071 3696 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-wv6mq I0511 17:57:46.732856 3696 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-wv6mq] pod update took 15.806901ms I0511 17:57:46.732883 3696 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-wv6mq] addLogicalPort annotation time took 15.837576ms I0511 17:57:46.734192 3696 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-wv6mq] addLogicalPort took 17.245758ms, libovsdb time 921.008µs I0511 17:57:47.078543 3696 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-wv6mq 6eb07e4e8723c5436ad6a362a3823f935916becf5a84d3bba847e66ae849858b network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-wv6mq 6eb07e4e8723c5436ad6a362a3823f935916becf5a84d3bba847e66ae849858b network default NAD default NAD key ] I0511 17:57:47.088482 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-wv6mq, hostIfaceName: 6eb07e4e8723c54, network: default, NAD default, SandboxID: "6eb07e4e8723c5436ad6a362a3823f935916becf5a84d3bba847e66ae849858b", PCI device ID: , UID: "da11a58c-c6bd-4906-acaf-1aba5ddefa9d", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0511 17:57:47.135419 3696 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-wv6mq 6eb07e4e8723c5436ad6a362a3823f935916becf5a84d3bba847e66ae849858b network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-wv6mq 6eb07e4e8723c5436ad6a362a3823f935916becf5a84d3bba847e66ae849858b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:6e:a3:c1:5b:f8\",\"name\":\"6eb07e4e8723c54\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c78cfa1-6d93-4152-ba78-1d4843dedf68\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 17:57:51.128768 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0511 17:57:52.190451 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-686f4894f4-q5bwh IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-686f4894f4-q5bwh: no pod IPs found W0511 17:57:52.196665 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-58f8556b85-j6jnr IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-58f8556b85-j6jnr: no pod IPs found W0511 17:57:52.206312 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0511 17:57:54.834818 3696 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-0-174.ec2.internal I0511 17:57:54.834841 3696 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-0-174.ec2.internal I0511 17:57:54.834933 3696 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-0-174.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-174.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-174.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-174.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-174.ec2.internal I0511 17:57:55.924692 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9c57r I0511 17:57:55.924722 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9c57r I0511 17:57:55.924747 3696 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-whd57 I0511 17:57:55.924764 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-46nhv I0511 17:57:55.924780 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-46nhv I0511 17:57:55.924782 3696 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-9c57r, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9c57r: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9c57r: no pod IPs found I0511 17:57:55.924807 3696 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-46nhv, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-46nhv: suppressed error logged: pod openshift-multus/network-metrics-daemon-46nhv: no pod IPs found I0511 17:57:55.924768 3696 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-whd57 I0511 17:57:55.924829 3696 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-whd57, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-whd57: suppressed error logged: pod kube-system/global-pull-secret-syncer-whd57: no pod IPs found I0511 17:58:01.092211 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:01.883207 3696 cni.go:452] [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key ] I0511 17:58:01.892746 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-58f8556b85-pnznl, hostIfaceName: c4f691ffcd2eb91, network: default, NAD default, SandboxID: "c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e", PCI device ID: , UID: "de0fc9b1-16eb-4338-b43e-81ca505b3c85", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0511 17:58:01.938934 3696 cni.go:452] [openshift-dns/dns-default-khg57 4fcde9d8cb92791899d23167d9fcdf4ef54580e44a36569e3b3f1bc734d105a2 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-khg57 4fcde9d8cb92791899d23167d9fcdf4ef54580e44a36569e3b3f1bc734d105a2 network default NAD default NAD key ] I0511 17:58:01.944203 3696 cni.go:473] [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:41:6c:46:e8:13\",\"name\":\"c4f691ffcd2eb91\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/30d87706-5366-4bfe-9135-1c6b449641b5\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:01.949217 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-khg57, hostIfaceName: 4fcde9d8cb92791, network: default, NAD default, SandboxID: "4fcde9d8cb92791899d23167d9fcdf4ef54580e44a36569e3b3f1bc734d105a2", PCI device ID: , UID: "0a847e6b-08b4-4f92-bd72-43d41badf7f3", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0511 17:58:01.967524 3696 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf cb0e0e113bfb35975b0ecdbfe2ceadde565dc388592f054f2bf3554734147bb8 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf cb0e0e113bfb35975b0ecdbfe2ceadde565dc388592f054f2bf3554734147bb8 network default NAD default NAD key ] I0511 17:58:01.976548 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-gwxgf, hostIfaceName: cb0e0e113bfb359, network: default, NAD default, SandboxID: "cb0e0e113bfb35975b0ecdbfe2ceadde565dc388592f054f2bf3554734147bb8", PCI device ID: , UID: "a69d1ad9-6244-4767-91b2-c904e405a1cd", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0511 17:58:01.998304 3696 cni.go:473] [openshift-dns/dns-default-khg57 4fcde9d8cb92791899d23167d9fcdf4ef54580e44a36569e3b3f1bc734d105a2 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-khg57 4fcde9d8cb92791899d23167d9fcdf4ef54580e44a36569e3b3f1bc734d105a2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:44:8c:60:c0:e2\",\"name\":\"4fcde9d8cb92791\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad735638-d2be-4603-8ca8-741451540a0d\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:02.026630 3696 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf cb0e0e113bfb35975b0ecdbfe2ceadde565dc388592f054f2bf3554734147bb8 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-gwxgf cb0e0e113bfb35975b0ecdbfe2ceadde565dc388592f054f2bf3554734147bb8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:57:df:2b:3e:ef\",\"name\":\"cb0e0e113bfb359\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e90ebcf9-de1b-4a57-a152-ea02a632c4a8\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:02.040172 3696 cni.go:452] [openshift-ingress/router-default-77758cd455-8z89h 6e5e8c1b979ef3bb34c7505cabd028b91d76a2364312df8d2d37cb36fbb4d36d network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-77758cd455-8z89h 6e5e8c1b979ef3bb34c7505cabd028b91d76a2364312df8d2d37cb36fbb4d36d network default NAD default NAD key ] I0511 17:58:02.050372 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-77758cd455-8z89h, hostIfaceName: 6e5e8c1b979ef3b, network: default, NAD default, SandboxID: "6e5e8c1b979ef3bb34c7505cabd028b91d76a2364312df8d2d37cb36fbb4d36d", PCI device ID: , UID: "5555ccea-0961-4ac9-873c-b1a76fb43502", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0511 17:58:02.067583 3696 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w c4f14b597b732aadfd93f9fa0c3a3bebe64bc9fd31d1891abe00320ece4d185f network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w c4f14b597b732aadfd93f9fa0c3a3bebe64bc9fd31d1891abe00320ece4d185f network default NAD default NAD key ] I0511 17:58:02.077598 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-ptg2w, hostIfaceName: c4f14b597b732aa, network: default, NAD default, SandboxID: "c4f14b597b732aadfd93f9fa0c3a3bebe64bc9fd31d1891abe00320ece4d185f", PCI device ID: , UID: "ad5b1c14-d6b4-44a1-a29b-5a5d1dd57895", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0511 17:58:02.092258 3696 cni.go:452] [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key ] I0511 17:58:02.103485 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5f7f58d6b5-kmnkj, hostIfaceName: aa88bd3bcadd490, network: default, NAD default, SandboxID: "aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729", PCI device ID: , UID: "3f00536b-7e74-455b-bf11-d6e5a1005ed0", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0511 17:58:02.108999 3696 cni.go:473] [openshift-ingress/router-default-77758cd455-8z89h 6e5e8c1b979ef3bb34c7505cabd028b91d76a2364312df8d2d37cb36fbb4d36d network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-77758cd455-8z89h 6e5e8c1b979ef3bb34c7505cabd028b91d76a2364312df8d2d37cb36fbb4d36d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:4b:e1:03:a6:5e\",\"name\":\"6e5e8c1b979ef3b\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/48bd02d8-668d-46b2-9c7f-0e8fb38f98bf\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:02.136415 3696 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w c4f14b597b732aadfd93f9fa0c3a3bebe64bc9fd31d1891abe00320ece4d185f network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-ptg2w c4f14b597b732aadfd93f9fa0c3a3bebe64bc9fd31d1891abe00320ece4d185f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:a5:4d:dc:e6:49\",\"name\":\"c4f14b597b732aa\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0c2f332-b035-49c2-b29b-cf76ffafac0d\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:02.137088 3696 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv 9325499a6b93e7e092907a2467988dae6513a5c2571c40b4bf8e335506cfa9ff network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv 9325499a6b93e7e092907a2467988dae6513a5c2571c40b4bf8e335506cfa9ff network default NAD default NAD key ] I0511 17:58:02.139242 3696 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g 75136cc4187db15aa2a0ba2ce9198a62f99e3c371bde7d44a118a5571b125046 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g 75136cc4187db15aa2a0ba2ce9198a62f99e3c371bde7d44a118a5571b125046 network default NAD default NAD key ] I0511 17:58:02.159970 3696 cni.go:473] [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:06:55:bc:78:11\",\"name\":\"aa88bd3bcadd490\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b01579f9-4f41-43cf-8311-d25cd7c3366c\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:02.162101 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-vwvnv, hostIfaceName: 9325499a6b93e7e, network: default, NAD default, SandboxID: "9325499a6b93e7e092907a2467988dae6513a5c2571c40b4bf8e335506cfa9ff", PCI device ID: , UID: "324afbbb-ce06-444b-b132-418627759051", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0511 17:58:02.162262 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-4c49g, hostIfaceName: 75136cc4187db15, network: default, NAD default, SandboxID: "75136cc4187db15aa2a0ba2ce9198a62f99e3c371bde7d44a118a5571b125046", PCI device ID: , UID: "003cd4fb-bae5-4789-9b90-910dc63cf67d", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0511 17:58:02.219258 3696 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv 9325499a6b93e7e092907a2467988dae6513a5c2571c40b4bf8e335506cfa9ff network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-vwvnv 9325499a6b93e7e092907a2467988dae6513a5c2571c40b4bf8e335506cfa9ff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:ad:31:86:63:da\",\"name\":\"9325499a6b93e7e\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/86b621d9-37cb-400a-9202-f2d5362af867\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:02.237048 3696 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g 75136cc4187db15aa2a0ba2ce9198a62f99e3c371bde7d44a118a5571b125046 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-4c49g 75136cc4187db15aa2a0ba2ce9198a62f99e3c371bde7d44a118a5571b125046 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:18:45:cd:c0:e6\",\"name\":\"75136cc4187db15\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bdf3054e-ea46-49c8-b54c-daa8231a6d42\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:04.090908 3696 cni.go:452] [openshift-multus/network-metrics-daemon-hn9lj a25a8c3d00235f94e268522f31f0a39292fdf1523d3ca4573f4e1424f6b7974e network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-hn9lj a25a8c3d00235f94e268522f31f0a39292fdf1523d3ca4573f4e1424f6b7974e network default NAD default NAD key ] I0511 17:58:04.101408 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-hn9lj, hostIfaceName: a25a8c3d00235f9, network: default, NAD default, SandboxID: "a25a8c3d00235f94e268522f31f0a39292fdf1523d3ca4573f4e1424f6b7974e", PCI device ID: , UID: "3a2b3bbc-e3c1-412d-ac2e-c68da28d8db0", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0511 17:58:04.169021 3696 cni.go:473] [openshift-multus/network-metrics-daemon-hn9lj a25a8c3d00235f94e268522f31f0a39292fdf1523d3ca4573f4e1424f6b7974e network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-hn9lj a25a8c3d00235f94e268522f31f0a39292fdf1523d3ca4573f4e1424f6b7974e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:85:d1:c6:84:e4\",\"name\":\"a25a8c3d00235f9\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/664da10d-ddfe-450e-b48f-2b76120d1ed1\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 17:58:11.404672 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0511 17:58:11.450977 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:58:11.499786 3696 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.499804 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.499809 3696 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0511 17:58:11.535375 3696 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0511 17:58:11.549859 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:11.597875 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:11.610674 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-7tfnp for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:11.610803 3696 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp I0511 17:58:11.625621 3696 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp] pod update took 14.880648ms I0511 17:58:11.625642 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp] addLogicalPort annotation time took 14.908906ms I0511 17:58:11.627025 3696 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp] addLogicalPort took 16.366521ms, libovsdb time 1.027935ms I0511 17:58:11.706059 3696 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-w6ztv] creating logical port openshift-insights_insights-runtime-extractor-w6ztv for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:11.706171 3696 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-w6ztv W0511 17:58:11.718288 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:11.723598 3696 pod.go:62] [openshift-insights/insights-runtime-extractor-w6ztv] pod update took 17.452567ms I0511 17:58:11.723621 3696 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-w6ztv] addLogicalPort annotation time took 17.478102ms I0511 17:58:11.724819 3696 pods.go:271] [openshift-insights/insights-runtime-extractor-w6ztv] addLogicalPort took 18.779132ms, libovsdb time 856.004µs W0511 17:58:11.748324 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-5d5d54496c-swlws IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-5d5d54496c-swlws: no pod IPs found I0511 17:58:11.777685 3696 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-kgbxc] creating logical port openshift-console_downloads-5c54f4944f-kgbxc for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:11.777830 3696 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-kgbxc I0511 17:58:11.800140 3696 pod.go:62] [openshift-console/downloads-5c54f4944f-kgbxc] pod update took 22.345709ms I0511 17:58:11.800165 3696 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-kgbxc] addLogicalPort annotation time took 22.375946ms I0511 17:58:11.801541 3696 pods.go:271] [openshift-console/downloads-5c54f4944f-kgbxc] addLogicalPort took 23.875217ms, libovsdb time 924.101µs I0511 17:58:11.837493 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-174.ec2.internal in network "default" I0511 17:58:11.837564 3696 node_tracker.go:235] Node ip-10-0-0-174.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-174.ec2.internal" I0511 17:58:11.837578 3696 node_tracker.go:169] Node ip-10-0-0-174.ec2.internal switch + router changed, syncing services in network "default" I0511 17:58:11.837588 3696 services_controller.go:574] Full service sync requested for network=default I0511 17:58:11.837513 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "ip-10-0-0-174.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:58:11.837722 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0511 17:58:11.837874 3696 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.837890 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.837896 3696 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.837950 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:11.875030 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "ip-10-0-0-174.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:11.985428 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-174.ec2.internal in network "default" I0511 17:58:11.985460 3696 default_network_controller.go:1025] Node "ip-10-0-0-174.ec2.internal" in remote zone "ip-10-0-0-174.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:11.985485 3696 node_tracker.go:169] Node ip-10-0-0-174.ec2.internal switch + router changed, syncing services in network "default" I0511 17:58:11.985509 3696 services_controller.go:574] Full service sync requested for network=default W0511 17:58:11.985576 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0511 17:58:11.985843 3696 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.985854 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.985858 3696 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:11.985907 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:11.986513 3696 zone_ic_handler.go:195] Time taken to create transit switch: 308.582µs I0511 17:58:11.986532 3696 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-174.ec2.internal for the network default I0511 17:58:11.987679 3696 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-174.ec2.internal" on network "default" took: 1.503332ms I0511 17:58:12.034446 3696 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp 1451dc4f1956d9a2018bf77cab6008a86981596edbe879f8f215163efc5bbbaa network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp 1451dc4f1956d9a2018bf77cab6008a86981596edbe879f8f215163efc5bbbaa network default NAD default NAD key ] I0511 17:58:12.044126 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-7tfnp, hostIfaceName: 1451dc4f1956d9a, network: default, NAD default, SandboxID: "1451dc4f1956d9a2018bf77cab6008a86981596edbe879f8f215163efc5bbbaa", PCI device ID: , UID: "99cb0d25-818b-42d6-a0bd-82698a6ecb3c", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0511 17:58:12.098237 3696 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp 1451dc4f1956d9a2018bf77cab6008a86981596edbe879f8f215163efc5bbbaa network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7tfnp 1451dc4f1956d9a2018bf77cab6008a86981596edbe879f8f215163efc5bbbaa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:52:9f:e9:03:12\",\"name\":\"1451dc4f1956d9a\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eeb675b0-ffb7-4e9e-87d6-128e37b657c8\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:12.177895 3696 cni.go:452] [openshift-console/downloads-5c54f4944f-kgbxc 6963c11e49370be3a7078a957424ee04a46ff4e0bd317e747b647bc41e5cf52f network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-kgbxc 6963c11e49370be3a7078a957424ee04a46ff4e0bd317e747b647bc41e5cf52f network default NAD default NAD key ] I0511 17:58:12.187577 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-kgbxc, hostIfaceName: 6963c11e49370be, network: default, NAD default, SandboxID: "6963c11e49370be3a7078a957424ee04a46ff4e0bd317e747b647bc41e5cf52f", PCI device ID: , UID: "667712d8-8967-4d0b-ab16-875546f35a00", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0511 17:58:12.236529 3696 cni.go:473] [openshift-console/downloads-5c54f4944f-kgbxc 6963c11e49370be3a7078a957424ee04a46ff4e0bd317e747b647bc41e5cf52f network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-kgbxc 6963c11e49370be3a7078a957424ee04a46ff4e0bd317e747b647bc41e5cf52f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:da:0f:e7:70:6d\",\"name\":\"6963c11e49370be\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f75879f5-09c0-4387-9bc3-563b0229a8a1\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:12.421462 3696 cni.go:452] [openshift-insights/insights-runtime-extractor-w6ztv 1fd57da73d8d0efe599557aa58e2e9e9ce1a5e20e86dd12e4b6144bedeb50d48 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-w6ztv 1fd57da73d8d0efe599557aa58e2e9e9ce1a5e20e86dd12e4b6144bedeb50d48 network default NAD default NAD key ] I0511 17:58:12.431461 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-w6ztv, hostIfaceName: 1fd57da73d8d0ef, network: default, NAD default, SandboxID: "1fd57da73d8d0efe599557aa58e2e9e9ce1a5e20e86dd12e4b6144bedeb50d48", PCI device ID: , UID: "7a823b08-963b-4b12-be0c-27c9ee7019d7", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0511 17:58:12.480612 3696 cni.go:473] [openshift-insights/insights-runtime-extractor-w6ztv 1fd57da73d8d0efe599557aa58e2e9e9ce1a5e20e86dd12e4b6144bedeb50d48 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-w6ztv 1fd57da73d8d0efe599557aa58e2e9e9ce1a5e20e86dd12e4b6144bedeb50d48 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:82:f2:5c:e9:2b\",\"name\":\"1fd57da73d8d0ef\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/95bb1535-58e4-41be-8ab1-6cbadf714d8e\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:12.544121 3696 egressfirewall.go:723] Syncing node "ip-10-0-0-174.ec2.internal" for egress firewall I0511 17:58:12.544148 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-174.ec2.internal in network "default" I0511 17:58:12.544205 3696 node_tracker.go:169] Node ip-10-0-0-174.ec2.internal switch + router changed, syncing services in network "default" I0511 17:58:12.544220 3696 services_controller.go:574] Full service sync requested for network=default I0511 17:58:12.544152 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-174.ec2.internal I0511 17:58:12.544227 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-174.ec2.internal", IP: 10.0.0.174 to PMTUD blocking rules I0511 17:58:12.544242 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-174.ec2.internal: 93.576µs I0511 17:58:12.544246 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-174.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0511 17:58:12.544315 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0511 17:58:12.544539 3696 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:12.544555 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:12.544561 3696 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:12.544613 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:12.550277 3696 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-174.ec2.internal. Updating no re-route policies I0511 17:58:12.562326 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-174.ec2.internal", IP: 10.0.0.174 to PMTUD blocking rules I0511 17:58:12.562347 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-174.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0511 17:58:14.081812 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0511 17:58:14.211718 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:14.211739 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:14.211745 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0511 17:58:14.238633 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0511 17:58:14.262886 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0511 17:58:14.416854 3696 base_network_controller_pods.go:487] [default/openshift-console/console-7d6559f477-8qr5l] creating logical port openshift-console_console-7d6559f477-8qr5l for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:14.416997 3696 kube.go:256] Updating pod openshift-console/console-7d6559f477-8qr5l I0511 17:58:14.430564 3696 pod.go:62] [openshift-console/console-7d6559f477-8qr5l] pod update took 13.59756ms I0511 17:58:14.430583 3696 base_network_controller_pods.go:951] [default/openshift-console/console-7d6559f477-8qr5l] addLogicalPort annotation time took 13.620839ms I0511 17:58:14.432005 3696 pods.go:271] [openshift-console/console-7d6559f477-8qr5l] addLogicalPort took 15.170247ms, libovsdb time 1.050449ms I0511 17:58:14.815798 3696 cni.go:452] [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key ] I0511 17:58:14.825943 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7d6559f477-8qr5l, hostIfaceName: aeaff49bdcab392, network: default, NAD default, SandboxID: "aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed", PCI device ID: , UID: "fa39bde8-fb60-44bb-9a75-02801b92a401", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0511 17:58:14.878434 3696 cni.go:473] [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:5c:45:15:49:8a\",\"name\":\"aeaff49bdcab392\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ccb05c39-dcea-4b04-bce4-d7785faf681f\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:16.010371 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-174.ec2.internal I0511 17:58:16.010443 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-174.ec2.internal: 80.554µs W0511 17:58:17.364626 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0511 17:58:19.269148 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0511 17:58:21.443191 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0511 17:58:22.423582 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0511 17:58:27.529032 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0511 17:58:27.529096 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0511 17:58:28.576863 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:28.577004 3696 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0511 17:58:28.613899 3696 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 36.931802ms I0511 17:58:28.613923 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 36.962349ms I0511 17:58:28.615325 3696 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 38.479974ms, libovsdb time 1.034984ms I0511 17:58:29.428170 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs] creating logical port openshift-monitoring_thanos-querier-6dfbd7ff89-j7dzs for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:29.428281 3696 kube.go:256] Updating pod openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs I0511 17:58:29.445201 3696 pod.go:62] [openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs] pod update took 16.95725ms I0511 17:58:29.445228 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs] addLogicalPort annotation time took 16.986154ms I0511 17:58:29.446612 3696 pods.go:271] [openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs] addLogicalPort took 18.459477ms, libovsdb time 894.632µs I0511 17:58:31.673242 3696 cni.go:452] [openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs 40afe79e3b2b2cd97c4da8afdce35104d29229cb1d1bb6754fb9e826779f47fd network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs 40afe79e3b2b2cd97c4da8afdce35104d29229cb1d1bb6754fb9e826779f47fd network default NAD default NAD key ] I0511 17:58:31.674291 3696 cni.go:452] [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key ] I0511 17:58:31.683882 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 5aa97fc3f34bd98, network: default, NAD default, SandboxID: "5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527", PCI device ID: , UID: "608e66cc-171d-49cc-b7d6-21c2a5694db5", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0511 17:58:31.684433 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-6dfbd7ff89-j7dzs, hostIfaceName: 40afe79e3b2b2cd, network: default, NAD default, SandboxID: "40afe79e3b2b2cd97c4da8afdce35104d29229cb1d1bb6754fb9e826779f47fd", PCI device ID: , UID: "0b567ca1-3dd0-44c7-85a7-6b37f89c6c56", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0511 17:58:31.736093 3696 cni.go:473] [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:8f:94:be:aa:df\",\"name\":\"5aa97fc3f34bd98\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2b652110-0c65-4c24-9730-428439986e0c\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 17:58:31.751951 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:31.755889 3696 cni.go:473] [openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs 40afe79e3b2b2cd97c4da8afdce35104d29229cb1d1bb6754fb9e826779f47fd network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-6dfbd7ff89-j7dzs 40afe79e3b2b2cd97c4da8afdce35104d29229cb1d1bb6754fb9e826779f47fd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:e7:74:25:6f:39\",\"name\":\"40afe79e3b2b2cd\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba932c83-d8de-4376-ac54-0c92ffafb6db\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:31.878463 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-648677c5c6-t6jd2] creating logical port openshift-monitoring_metrics-server-648677c5c6-t6jd2 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:31.878573 3696 kube.go:256] Updating pod openshift-monitoring/metrics-server-648677c5c6-t6jd2 I0511 17:58:31.902858 3696 pod.go:62] [openshift-monitoring/metrics-server-648677c5c6-t6jd2] pod update took 24.304181ms I0511 17:58:31.902885 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-648677c5c6-t6jd2] addLogicalPort annotation time took 24.34054ms I0511 17:58:31.904232 3696 pods.go:271] [openshift-monitoring/metrics-server-648677c5c6-t6jd2] addLogicalPort took 25.783027ms, libovsdb time 919.84µs W0511 17:58:32.148559 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-6984d4dd84-dnckg IP address from the namespace address-set, err: pod openshift-console/console-6984d4dd84-dnckg: no pod IPs found W0511 17:58:32.148596 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-6984d4dd84-sggvm IP address from the namespace address-set, err: pod openshift-console/console-6984d4dd84-sggvm: no pod IPs found I0511 17:58:32.154106 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-xxvcl for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:32.154215 3696 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl I0511 17:58:32.168811 3696 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl] pod update took 14.62219ms I0511 17:58:32.168830 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl] addLogicalPort annotation time took 14.645008ms I0511 17:58:32.170177 3696 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl] addLogicalPort took 16.084905ms, libovsdb time 909.286µs I0511 17:58:32.245689 3696 cni.go:452] [openshift-monitoring/metrics-server-648677c5c6-t6jd2 11ea4b29f912ecf5ab40ffa3fd8973111a6ac440c8d14b903f598b7e372c011e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-648677c5c6-t6jd2 11ea4b29f912ecf5ab40ffa3fd8973111a6ac440c8d14b903f598b7e372c011e network default NAD default NAD key ] I0511 17:58:32.259642 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-648677c5c6-t6jd2, hostIfaceName: 11ea4b29f912ecf, network: default, NAD default, SandboxID: "11ea4b29f912ecf5ab40ffa3fd8973111a6ac440c8d14b903f598b7e372c011e", PCI device ID: , UID: "fdfc4bc6-13fd-4c82-abc7-d31e8b17602c", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0511 17:58:32.320345 3696 cni.go:473] [openshift-monitoring/metrics-server-648677c5c6-t6jd2 11ea4b29f912ecf5ab40ffa3fd8973111a6ac440c8d14b903f598b7e372c011e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-648677c5c6-t6jd2 11ea4b29f912ecf5ab40ffa3fd8973111a6ac440c8d14b903f598b7e372c011e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:2c:b2:54:90:f1\",\"name\":\"11ea4b29f912ecf\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6fad41e4-fa3e-4210-9bfe-4fc2985914a0\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 17:58:32.343987 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0511 17:58:32.375850 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0511 17:58:32.535797 3696 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl 3d8a0e5e533a0a34ecd9d5069803ac22ec55b6feaec84aca9c2474156a4bd115 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl 3d8a0e5e533a0a34ecd9d5069803ac22ec55b6feaec84aca9c2474156a4bd115 network default NAD default NAD key ] I0511 17:58:32.550103 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-xxvcl, hostIfaceName: 3d8a0e5e533a0a3, network: default, NAD default, SandboxID: "3d8a0e5e533a0a34ecd9d5069803ac22ec55b6feaec84aca9c2474156a4bd115", PCI device ID: , UID: "3c3b5c45-4a2f-46d5-9b66-fc84c723f7cd", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0511 17:58:32.624873 3696 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl 3d8a0e5e533a0a34ecd9d5069803ac22ec55b6feaec84aca9c2474156a4bd115 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-xxvcl 3d8a0e5e533a0a34ecd9d5069803ac22ec55b6feaec84aca9c2474156a4bd115 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:28:b1:de:ed:cb\",\"name\":\"3d8a0e5e533a0a3\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96044c98-3832-4aba-81c1-8e7afd80ba9e\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:33.748142 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:33.748327 3696 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0511 17:58:33.769814 3696 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 21.522079ms I0511 17:58:33.769835 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 21.549295ms I0511 17:58:33.771614 3696 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 23.49084ms, libovsdb time 1.181844ms I0511 17:58:34.235602 3696 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0511 17:58:34.321877 3696 cni.go:452] [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key ] I0511 17:58:34.333086 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 9e4af81c2cc6e13, network: default, NAD default, SandboxID: "9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7", PCI device ID: , UID: "8acb909c-e48e-4fdd-ab06-ae2e4de33daa", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0511 17:58:34.391354 3696 cni.go:473] [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:1c:a9:3c:61:8b\",\"name\":\"9e4af81c2cc6e13\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d41454d-cfef-40e6-8279-45729ff24886\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 17:58:35.376869 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0511 17:58:35.422232 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:58:35.422295 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:58:35.462628 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:35.462652 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:35.462661 3696 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:35.489564 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:35.489589 3696 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:35.489595 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:36.554800 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:58:36.554875 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:58:36.618530 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0511 17:58:37.090162 3696 cni.go:452] [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key ] I0511 17:58:37.174247 3696 cni.go:473] [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-58f8556b85-pnznl c4f691ffcd2eb919e281a7f126e223efaa7a7960c88614359a788b4a53193a7e network default NAD default NAD key default], result "{}", err I0511 17:58:37.255636 3696 obj_retry.go:498] Detected object openshift-image-registry/image-registry-58f8556b85-pnznl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:58:37.255675 3696 pods.go:176] Deleting pod: openshift-image-registry/image-registry-58f8556b85-pnznl I0511 17:58:37.257333 3696 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-58f8556b85-pnznl, ips: 10.128.0.6 I0511 17:58:37.257370 3696 obj_retry.go:498] Detected object openshift-image-registry/image-registry-58f8556b85-pnznl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:58:37.267109 3696 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6f88959578-bpdpr] creating logical port openshift-image-registry_image-registry-6f88959578-bpdpr for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:37.267212 3696 kube.go:256] Updating pod openshift-image-registry/image-registry-6f88959578-bpdpr W0511 17:58:37.269719 3696 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0511 17:58:37.293676 3696 pod.go:62] [openshift-image-registry/image-registry-6f88959578-bpdpr] pod update took 26.495172ms I0511 17:58:37.293756 3696 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6f88959578-bpdpr] addLogicalPort annotation time took 26.583048ms I0511 17:58:37.295371 3696 pods.go:271] [openshift-image-registry/image-registry-6f88959578-bpdpr] addLogicalPort took 28.278758ms, libovsdb time 1.105519ms I0511 17:58:37.362618 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-234.ec2.internal in network "default" I0511 17:58:37.362646 3696 node_tracker.go:220] Node ip-10-0-1-234.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:58:37.362695 3696 egressfirewall.go:723] Syncing node "ip-10-0-1-234.ec2.internal" for egress firewall I0511 17:58:37.362704 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-234.ec2.internal I0511 17:58:37.362721 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-234.ec2.internal: 21.03µs E0511 17:58:37.362739 3696 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-234.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:58:37.362755 3696 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-234.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-234.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-234.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-234.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-234.ec2.internal, error parsing annotation for node ip-10-0-1-234.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0511 17:58:37.374066 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-234.ec2.internal annotation: unexpected end of JSON input I0511 17:58:37.374073 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0511 17:58:37.374111 3696 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-234.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0511 17:58:37.386019 3696 egressfirewall.go:723] Syncing node "ip-10-0-1-234.ec2.internal" for egress firewall I0511 17:58:37.386048 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-234.ec2.internal E0511 17:58:37.386060 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-234.ec2.internal annotation: unexpected end of JSON input I0511 17:58:37.386065 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-234.ec2.internal: 22.242µs I0511 17:58:37.386091 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0511 17:58:37.386125 3696 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-234.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0511 17:58:37.412865 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-234.ec2.internal annotation: unexpected end of JSON input E0511 17:58:37.412944 3696 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-234.ec2.internal annotation: unexpected end of JSON input I0511 17:58:37.412962 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0511 17:58:37.413015 3696 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-234.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-234.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-234.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-234.ec2.internal" I0511 17:58:37.433471 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true W0511 17:58:37.476779 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:37.476807 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:37.476815 3696 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:37.476820 3696 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0511 17:58:37.492219 3696 obj_retry.go:492] Detected object openshift-image-registry/image-registry-58f8556b85-pnznl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:58:37.492241 3696 obj_retry.go:492] Detected object openshift-image-registry/image-registry-58f8556b85-pnznl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:58:37.503013 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-58f8556b85-pnznl I0511 17:58:37.503036 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-58f8556b85-pnznl I0511 17:58:37.603919 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:37.603980 3696 egressfirewall.go:723] Syncing node "ip-10-0-1-234.ec2.internal" for egress firewall I0511 17:58:37.603995 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-234.ec2.internal I0511 17:58:37.604072 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-234.ec2.internal: 82.198µs I0511 17:58:37.626990 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:37.678286 3696 cni.go:452] [openshift-image-registry/image-registry-6f88959578-bpdpr 79acd017bc9cadb3a885f8ceb8f4611ca26a070db03bacf57341b2e341e7d336 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6f88959578-bpdpr 79acd017bc9cadb3a885f8ceb8f4611ca26a070db03bacf57341b2e341e7d336 network default NAD default NAD key ] I0511 17:58:37.691590 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6f88959578-bpdpr, hostIfaceName: 79acd017bc9cadb, network: default, NAD default, SandboxID: "79acd017bc9cadb3a885f8ceb8f4611ca26a070db03bacf57341b2e341e7d336", PCI device ID: , UID: "72f0ff6d-765e-4182-8d98-4473aa2ba475", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] W0511 17:58:37.697395 3696 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:37.697419 3696 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:37.697426 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:37.697433 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0511 17:58:37.759588 3696 cni.go:473] [openshift-image-registry/image-registry-6f88959578-bpdpr 79acd017bc9cadb3a885f8ceb8f4611ca26a070db03bacf57341b2e341e7d336 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6f88959578-bpdpr 79acd017bc9cadb3a885f8ceb8f4611ca26a070db03bacf57341b2e341e7d336 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:54:b1:9e:8c:a3\",\"name\":\"79acd017bc9cadb\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b37dc39-ebf6-47e6-ab1b-3e011e8b9680\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:37.815056 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:58:38.022756 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-66f48446bc-8pvtm IP address from the namespace address-set, err: pod openshift-console/console-66f48446bc-8pvtm: no pod IPs found W0511 17:58:38.039586 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-66f48446bc-ctwgs IP address from the namespace address-set, err: pod openshift-console/console-66f48446bc-ctwgs: no pod IPs found I0511 17:58:41.596669 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:58:43.406923 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:43.406945 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:43.406951 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0511 17:58:43.406958 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0511 17:58:47.535907 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:47.592948 3696 cni.go:452] [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key ] I0511 17:58:47.689515 3696 cni.go:473] [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7d6559f477-8qr5l aeaff49bdcab392bee6dfa003568c67ea2a46d3586d89499832002ef94860eed network default NAD default NAD key default], result "{}", err I0511 17:58:47.767274 3696 obj_retry.go:498] Detected object openshift-console/console-7d6559f477-8qr5l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:58:47.767314 3696 pods.go:176] Deleting pod: openshift-console/console-7d6559f477-8qr5l I0511 17:58:47.768580 3696 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7d6559f477-8qr5l, ips: 10.128.0.27 I0511 17:58:47.768605 3696 obj_retry.go:498] Detected object openshift-console/console-7d6559f477-8qr5l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:58:47.775568 3696 base_network_controller_pods.go:487] [default/openshift-console/console-75d7f46b9f-bbztb] creating logical port openshift-console_console-75d7f46b9f-bbztb for pod on switch ip-10-0-2-22.ec2.internal I0511 17:58:47.775683 3696 kube.go:256] Updating pod openshift-console/console-75d7f46b9f-bbztb I0511 17:58:47.792026 3696 pod.go:62] [openshift-console/console-75d7f46b9f-bbztb] pod update took 16.366551ms I0511 17:58:47.792053 3696 base_network_controller_pods.go:951] [default/openshift-console/console-75d7f46b9f-bbztb] addLogicalPort annotation time took 16.397767ms I0511 17:58:47.793359 3696 pods.go:271] [openshift-console/console-75d7f46b9f-bbztb] addLogicalPort took 17.805948ms, libovsdb time 800.513µs I0511 17:58:48.190875 3696 cni.go:452] [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key ] I0511 17:58:48.200282 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-75d7f46b9f-bbztb, hostIfaceName: c95e809fdcc46d9, network: default, NAD default, SandboxID: "c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27", PCI device ID: , UID: "26518f46-f542-42bc-ae6f-bc1bc14d9145", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0511 17:58:48.251380 3696 cni.go:473] [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:3e:d6:ca:c9:f1\",\"name\":\"c95e809fdcc46d9\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70ca5b80-5d74-4eea-aac7-75ddf4594adf\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:58:48.462278 3696 obj_retry.go:492] Detected object openshift-console/console-7d6559f477-8qr5l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:58:48.462306 3696 obj_retry.go:492] Detected object openshift-console/console-7d6559f477-8qr5l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:58:48.469574 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d6559f477-8qr5l I0511 17:58:48.469604 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d6559f477-8qr5l W0511 17:58:54.496883 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0511 17:58:54.496907 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0511 17:58:54.496959 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0511 17:58:54.834441 3696 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-1-234.ec2.internal I0511 17:58:54.834470 3696 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-1-234.ec2.internal I0511 17:58:54.834585 3696 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-1-234.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-234.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-234.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-234.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-234.ec2.internal W0511 17:58:55.510221 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0511 17:58:55.925138 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nvz94 I0511 17:58:55.925151 3696 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-g88kv I0511 17:58:55.925167 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nvz94 I0511 17:58:55.925164 3696 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-dsz9j I0511 17:58:55.925175 3696 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-g88kv I0511 17:58:55.925181 3696 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-dsz9j I0511 17:58:55.925206 3696 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-nvz94, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nvz94: suppressed error logged: pod openshift-multus/network-metrics-daemon-nvz94: no pod IPs found I0511 17:58:55.925211 3696 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-g88kv, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-g88kv: suppressed error logged: pod kube-system/global-pull-secret-syncer-g88kv: no pod IPs found I0511 17:58:55.925213 3696 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-dsz9j, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-dsz9j: suppressed error logged: pod openshift-network-diagnostics/network-check-target-dsz9j: no pod IPs found I0511 17:58:56.284543 3696 egressfirewall.go:723] Syncing node "ip-10-0-1-234.ec2.internal" for egress firewall I0511 17:58:56.284574 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-234.ec2.internal I0511 17:58:56.284575 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:58:56.284587 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-234.ec2.internal: 17.087µs W0511 17:58:56.384656 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0511 17:58:57.723002 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:58:58.524999 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0511 17:58:59.528924 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0511 17:58:59.528978 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0511 17:58:59.528985 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0511 17:58:59.569518 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:59.569545 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:58:59.569554 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 17:59:00.428573 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0511 17:59:00.534206 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0511 17:59:00.976228 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-234.ec2.internal in network "default" I0511 17:59:00.976263 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "ip-10-0-1-234.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:59:00.976270 3696 node_tracker.go:235] Node ip-10-0-1-234.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-234.ec2.internal" I0511 17:59:00.976337 3696 node_tracker.go:169] Node ip-10-0-1-234.ec2.internal switch + router changed, syncing services in network "default" I0511 17:59:00.976348 3696 services_controller.go:574] Full service sync requested for network=default W0511 17:59:00.976732 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:59:00.976749 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0511 17:59:01.029913 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "ip-10-0-1-234.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0511 17:59:01.123935 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-234.ec2.internal in network "default" I0511 17:59:01.124000 3696 node_tracker.go:169] Node ip-10-0-1-234.ec2.internal switch + router changed, syncing services in network "default" I0511 17:59:01.124019 3696 services_controller.go:574] Full service sync requested for network=default I0511 17:59:01.124068 3696 default_network_controller.go:1025] Node "ip-10-0-1-234.ec2.internal" in remote zone "ip-10-0-1-234.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0511 17:59:01.124519 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:59:01.124533 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0511 17:59:01.125079 3696 zone_ic_handler.go:195] Time taken to create transit switch: 279.771µs I0511 17:59:01.125093 3696 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-234.ec2.internal for the network default I0511 17:59:01.126090 3696 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-234.ec2.internal" on network "default" took: 1.301988ms I0511 17:59:03.683625 3696 egressfirewall.go:723] Syncing node "ip-10-0-1-234.ec2.internal" for egress firewall I0511 17:59:03.683632 3696 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-234.ec2.internal in network "default" I0511 17:59:03.683707 3696 node_tracker.go:169] Node ip-10-0-1-234.ec2.internal switch + router changed, syncing services in network "default" I0511 17:59:03.683630 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-234.ec2.internal I0511 17:59:03.683715 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-234.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0511 17:59:03.683734 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-234.ec2.internal", IP: 10.0.1.234 to PMTUD blocking rules I0511 17:59:03.683734 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-234.ec2.internal: 106.11µs I0511 17:59:03.683723 3696 services_controller.go:574] Full service sync requested for network=default W0511 17:59:03.684025 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:59:03.684041 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0511 17:59:03.689946 3696 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-234.ec2.internal. Updating no re-route policies I0511 17:59:03.701601 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-234.ec2.internal", IP: 10.0.1.234 to PMTUD blocking rules I0511 17:59:03.701616 3696 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-234.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0511 17:59:07.147845 3696 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-234.ec2.internal I0511 17:59:07.147887 3696 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-234.ec2.internal: 53.241µs W0511 17:59:09.574420 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0511 17:59:09.574510 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0511 17:59:09.574560 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0511 17:59:10.588315 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0511 17:59:20.245597 3696 cni.go:452] [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key ] I0511 17:59:20.329213 3696 cni.go:473] [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-5f7f58d6b5-kmnkj aa88bd3bcadd49062187d5c276c96978114fab73487b666875d62f5e0267a729 network default NAD default NAD key default], result "{}", err I0511 17:59:20.392305 3696 obj_retry.go:498] Detected object openshift-ingress/router-default-5f7f58d6b5-kmnkj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:20.392341 3696 pods.go:176] Deleting pod: openshift-ingress/router-default-5f7f58d6b5-kmnkj I0511 17:59:20.393905 3696 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5f7f58d6b5-kmnkj, ips: 10.128.0.14 I0511 17:59:20.393934 3696 obj_retry.go:498] Detected object openshift-ingress/router-default-5f7f58d6b5-kmnkj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:20.621623 3696 obj_retry.go:492] Detected object openshift-ingress/router-default-5f7f58d6b5-kmnkj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:20.621652 3696 obj_retry.go:492] Detected object openshift-ingress/router-default-5f7f58d6b5-kmnkj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:20.627450 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5f7f58d6b5-kmnkj I0511 17:59:20.627480 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5f7f58d6b5-kmnkj I0511 17:59:23.887626 3696 obj_retry.go:498] Detected object openshift-console/console-7d6559f477-p77qr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:23.888135 3696 obj_retry.go:498] Detected object openshift-console/console-7d6559f477-p77qr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:24.737017 3696 obj_retry.go:492] Detected object openshift-console/console-7d6559f477-p77qr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:24.737040 3696 obj_retry.go:492] Detected object openshift-console/console-7d6559f477-p77qr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:24.741841 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d6559f477-p77qr I0511 17:59:24.741866 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d6559f477-p77qr I0511 17:59:26.846417 3696 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5d5d54496c-dbkvw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:26.847029 3696 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5d5d54496c-dbkvw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:27.771073 3696 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5d5d54496c-dbkvw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:27.771109 3696 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5d5d54496c-dbkvw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:27.776421 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5d5d54496c-dbkvw I0511 17:59:27.776459 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5d5d54496c-dbkvw W0511 17:59:34.731560 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 17:59:34.731580 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0511 17:59:45.112124 3696 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0511 17:59:49.203851 3696 cni.go:452] [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key ] I0511 17:59:49.286528 3696 cni.go:473] [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 5aa97fc3f34bd988a2c7c6760b92e60aaad1983bac7bbeb64d6d434a4033f527 network default NAD default NAD key default], result "{}", err I0511 17:59:49.347396 3696 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:49.347440 3696 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0511 17:59:49.348988 3696 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.128.0.28 I0511 17:59:49.349023 3696 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:49.772192 3696 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:49.772225 3696 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:49.778802 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0511 17:59:49.778833 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0511 17:59:49.817259 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:59:49.817363 3696 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0511 17:59:49.839974 3696 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 22.64699ms I0511 17:59:49.839996 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 22.673173ms I0511 17:59:49.841208 3696 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 23.96588ms, libovsdb time 832.18µs I0511 17:59:50.177092 3696 cni.go:452] [openshift-monitoring/alertmanager-main-1 2aa90b27a52d387edfd688a41b266143f1770980892dcccfe1c4054dfe5c9abe network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 2aa90b27a52d387edfd688a41b266143f1770980892dcccfe1c4054dfe5c9abe network default NAD default NAD key ] I0511 17:59:50.186236 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 2aa90b27a52d387, network: default, NAD default, SandboxID: "2aa90b27a52d387edfd688a41b266143f1770980892dcccfe1c4054dfe5c9abe", PCI device ID: , UID: "d010b139-ce88-4c42-9728-003e817e6875", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0511 17:59:50.236442 3696 cni.go:473] [openshift-monitoring/alertmanager-main-1 2aa90b27a52d387edfd688a41b266143f1770980892dcccfe1c4054dfe5c9abe network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 2aa90b27a52d387edfd688a41b266143f1770980892dcccfe1c4054dfe5c9abe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:0f:05:b0:ad:55\",\"name\":\"2aa90b27a52d387\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5502c172-c59f-48e6-a7cf-ac7f63fc6c3d\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 17:59:53.481295 3696 cni.go:452] [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key ] I0511 17:59:53.569949 3696 cni.go:473] [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 9e4af81c2cc6e139bba36ace8f61bfd00e5fa8737a148a32e4bf9cb2288781c7 network default NAD default NAD key default], result "{}", err I0511 17:59:53.632043 3696 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:53.632079 3696 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0511 17:59:53.633404 3696 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.128.0.32 I0511 17:59:53.633434 3696 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 17:59:53.807125 3696 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:53.807148 3696 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 17:59:53.813818 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0511 17:59:53.813839 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0511 17:59:53.862953 3696 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-22.ec2.internal I0511 17:59:53.863087 3696 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0511 17:59:53.886165 3696 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 23.116516ms I0511 17:59:53.886185 3696 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 23.142055ms I0511 17:59:53.887419 3696 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 24.483473ms, libovsdb time 850.026µs I0511 17:59:54.227146 3696 cni.go:452] [openshift-monitoring/prometheus-k8s-1 83eea7cde7916ab649459ee84a30e870078400273990131212c3e0d27068cf33 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 83eea7cde7916ab649459ee84a30e870078400273990131212c3e0d27068cf33 network default NAD default NAD key ] I0511 17:59:54.237065 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 83eea7cde7916ab, network: default, NAD default, SandboxID: "83eea7cde7916ab649459ee84a30e870078400273990131212c3e0d27068cf33", PCI device ID: , UID: "4b9e6d7c-5f3b-44ab-813b-a6433de0c6a7", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0511 17:59:54.289643 3696 cni.go:473] [openshift-monitoring/prometheus-k8s-1 83eea7cde7916ab649459ee84a30e870078400273990131212c3e0d27068cf33 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 83eea7cde7916ab649459ee84a30e870078400273990131212c3e0d27068cf33 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:ba:f4:75:fd:45\",\"name\":\"83eea7cde7916ab\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7b2c519-a91f-43ad-b6f2-4bd95ed30a1b\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:00:05.777515 3696 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642040-cxxtn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:05.778054 3696 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642040-cxxtn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:06.800007 3696 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642040-cxxtn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:06.800035 3696 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642040-cxxtn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0511 18:00:10.244788 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 18:00:10.244894 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0511 18:00:10.244928 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0511 18:00:12.207453 3696 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:12.208088 3696 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:12.965774 3696 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:12.965801 3696 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:12.972087 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0511 18:00:12.972112 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0511 18:00:16.814710 3696 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:16.815254 3696 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:16.815283 3696 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:00:16.815293 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0511 18:00:16.815305 3696 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:00:16.815314 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0511 18:00:17.839656 3696 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:17.839686 3696 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:17.839702 3696 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:17.839711 3696 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq45hs of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:22.362129 3696 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0511 18:00:22.372080 3696 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0511 18:00:30.283650 3696 cni.go:452] [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key ] I0511 18:00:30.358997 3696 cni.go:473] [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-75d7f46b9f-bbztb c95e809fdcc46d99e909b7997d0bc7c8d6a87793b5dfaf645db690f1217b6b27 network default NAD default NAD key default], result "{}", err I0511 18:00:30.410762 3696 obj_retry.go:498] Detected object openshift-console/console-75d7f46b9f-bbztb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:30.410797 3696 pods.go:176] Deleting pod: openshift-console/console-75d7f46b9f-bbztb I0511 18:00:30.412165 3696 pods.go:236] Attempting to release IPs for pod: openshift-console/console-75d7f46b9f-bbztb, ips: 10.128.0.34 I0511 18:00:30.412189 3696 obj_retry.go:498] Detected object openshift-console/console-75d7f46b9f-bbztb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:30.418797 3696 base_network_controller_pods.go:487] [default/openshift-console/console-87ff8fdcf-nbl4k] creating logical port openshift-console_console-87ff8fdcf-nbl4k for pod on switch ip-10-0-2-22.ec2.internal I0511 18:00:30.418895 3696 kube.go:256] Updating pod openshift-console/console-87ff8fdcf-nbl4k I0511 18:00:30.433908 3696 pod.go:62] [openshift-console/console-87ff8fdcf-nbl4k] pod update took 15.045598ms I0511 18:00:30.433929 3696 base_network_controller_pods.go:951] [default/openshift-console/console-87ff8fdcf-nbl4k] addLogicalPort annotation time took 15.072041ms I0511 18:00:30.435113 3696 pods.go:271] [openshift-console/console-87ff8fdcf-nbl4k] addLogicalPort took 16.330396ms, libovsdb time 826.712µs I0511 18:00:30.778611 3696 cni.go:452] [openshift-console/console-87ff8fdcf-nbl4k c7fba1ecf5c2f19a0b1109cc8894dd5341743054a27db25ed123ac501239f61a network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-87ff8fdcf-nbl4k c7fba1ecf5c2f19a0b1109cc8894dd5341743054a27db25ed123ac501239f61a network default NAD default NAD key ] I0511 18:00:30.787903 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-87ff8fdcf-nbl4k, hostIfaceName: c7fba1ecf5c2f19, network: default, NAD default, SandboxID: "c7fba1ecf5c2f19a0b1109cc8894dd5341743054a27db25ed123ac501239f61a", PCI device ID: , UID: "44d92c8f-af1a-4e4b-b557-4f316fe7117e", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0511 18:00:30.838082 3696 cni.go:473] [openshift-console/console-87ff8fdcf-nbl4k c7fba1ecf5c2f19a0b1109cc8894dd5341743054a27db25ed123ac501239f61a network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-87ff8fdcf-nbl4k c7fba1ecf5c2f19a0b1109cc8894dd5341743054a27db25ed123ac501239f61a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:f2:e7:4f:d5:e0\",\"name\":\"c7fba1ecf5c2f19\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/60c17497-5606-4255-8e07-72f5e09f9b67\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:00:31.005948 3696 obj_retry.go:492] Detected object openshift-console/console-75d7f46b9f-bbztb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:31.005978 3696 obj_retry.go:492] Detected object openshift-console/console-75d7f46b9f-bbztb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:31.012672 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-75d7f46b9f-bbztb I0511 18:00:31.012703 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-75d7f46b9f-bbztb W0511 18:00:41.056296 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0511 18:00:41.307073 3696 obj_retry.go:498] Detected object openshift-console/console-75d7f46b9f-mfwk5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:41.307609 3696 obj_retry.go:498] Detected object openshift-console/console-75d7f46b9f-mfwk5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:41.773454 3696 obj_retry.go:492] Detected object openshift-console/console-75d7f46b9f-mfwk5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:41.773481 3696 obj_retry.go:492] Detected object openshift-console/console-75d7f46b9f-mfwk5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:41.779520 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-75d7f46b9f-mfwk5 I0511 18:00:41.779551 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-75d7f46b9f-mfwk5 W0511 18:00:55.176219 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0511 18:00:55.176239 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0511 18:00:56.488291 3696 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:56.488871 3696 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:00:57.156757 3696 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:57.156784 3696 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:00:57.162845 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0511 18:00:57.162877 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0511 18:02:37.373522 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=134 I0511 18:03:12.489965 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0511 18:03:21.496846 3696 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0511 18:03:31.376400 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=85 I0511 18:03:33.484230 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0511 18:03:41.463360 3696 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0511 18:04:02.472842 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0511 18:04:46.381309 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=702 I0511 18:05:11.376910 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=72 I0511 18:05:12.503207 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0511 18:05:18.478407 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0511 18:05:24.502973 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 W0511 18:05:27.663992 3696 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0511 18:05:38.702884 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0511 18:05:38.702961 3696 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-7lbrq IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-7lbrq: no pod IPs found I0511 18:05:43.873731 3696 namespace.go:98] [tsf] adding namespace I0511 18:05:43.874627 3696 namespace.go:102] [tsf] adding namespace took 869.056µs I0511 18:05:43.891735 3696 namespace.go:142] [tsf] updating namespace I0511 18:05:43.910414 3696 namespace.go:142] [tsf] updating namespace I0511 18:05:47.607769 3696 namespace.go:98] [tsf-tas] adding namespace I0511 18:05:47.607926 3696 namespace.go:98] [rhtpa-operator] adding namespace I0511 18:05:47.608050 3696 namespace.go:98] [rhbk-operator] adding namespace I0511 18:05:47.608118 3696 namespace.go:98] [tsf-keycloak] adding namespace I0511 18:05:47.608728 3696 namespace.go:102] [tsf-tas] adding namespace took 932.629µs I0511 18:05:47.609522 3696 namespace.go:102] [rhtpa-operator] adding namespace took 1.578855ms I0511 18:05:47.610058 3696 namespace.go:102] [rhbk-operator] adding namespace took 1.992303ms I0511 18:05:47.610589 3696 namespace.go:102] [tsf-keycloak] adding namespace took 2.457596ms I0511 18:05:47.617139 3696 namespace.go:98] [konflux-operator] adding namespace I0511 18:05:47.617211 3696 namespace.go:98] [tsf-tpa] adding namespace I0511 18:05:47.617818 3696 namespace.go:102] [konflux-operator] adding namespace took 625.258µs I0511 18:05:47.618394 3696 namespace.go:102] [tsf-tpa] adding namespace took 1.159758ms I0511 18:05:47.620683 3696 namespace.go:98] [konflux-ui] adding namespace I0511 18:05:47.620722 3696 namespace.go:98] [cert-manager-operator] adding namespace I0511 18:05:47.621278 3696 namespace.go:102] [konflux-ui] adding namespace took 584.794µs I0511 18:05:47.621896 3696 namespace.go:102] [cert-manager-operator] adding namespace took 1.161151ms I0511 18:05:47.649684 3696 namespace.go:142] [tsf-tas] updating namespace I0511 18:05:47.688849 3696 namespace.go:142] [rhbk-operator] updating namespace I0511 18:05:47.691565 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:05:47.710850 3696 namespace.go:142] [tsf-tas] updating namespace I0511 18:05:47.711045 3696 namespace.go:142] [konflux-operator] updating namespace I0511 18:05:47.711089 3696 namespace.go:142] [rhtpa-operator] updating namespace I0511 18:05:47.735697 3696 namespace.go:142] [cert-manager-operator] updating namespace I0511 18:05:47.740387 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:05:47.740460 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:05:47.773161 3696 namespace.go:142] [tsf-tas] updating namespace I0511 18:05:47.803701 3696 namespace.go:142] [rhbk-operator] updating namespace I0511 18:05:47.860165 3696 namespace.go:142] [rhbk-operator] updating namespace I0511 18:05:47.870126 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:05:47.891872 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:05:47.928226 3696 namespace.go:142] [konflux-operator] updating namespace I0511 18:05:47.950825 3696 namespace.go:142] [konflux-operator] updating namespace I0511 18:05:47.980281 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:05:48.003553 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:05:48.022713 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:05:48.049778 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:05:48.097683 3696 namespace.go:142] [cert-manager-operator] updating namespace I0511 18:05:48.118766 3696 namespace.go:142] [cert-manager-operator] updating namespace I0511 18:05:48.132653 3696 namespace.go:142] [rhtpa-operator] updating namespace I0511 18:05:48.145350 3696 namespace.go:142] [rhtpa-operator] updating namespace I0511 18:05:48.945940 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:05:48.958324 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:05:56.947373 3696 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:05:56.947931 3696 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:05:56.947960 3696 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:05:56.947970 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0511 18:05:56.947979 3696 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:05:56.947983 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0511 18:05:57.973120 3696 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:57.973146 3696 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:57.973168 3696 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:57.973181 3696 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe47rs8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:58.049311 3696 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:05:58.049871 3696 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:05:58.049901 3696 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:05:58.049908 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0511 18:05:58.049917 3696 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:05:58.049921 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0511 18:05:58.462830 3696 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0511 18:05:59.074299 3696 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:59.074330 3696 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:59.074346 3696 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:05:59.074358 3696 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6pxml of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:01.940988 3696 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:01.941547 3696 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:01.941585 3696 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:01.941596 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0511 18:06:01.941606 3696 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:01.941610 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0511 18:06:02.865293 3696 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:02.865319 3696 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:02.865334 3696 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:02.865344 3696 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6egr8qh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:02.949824 3696 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:02.950353 3696 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:02.950380 3696 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:02.950388 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0511 18:06:02.950396 3696 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:02.950400 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0511 18:06:03.983536 3696 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:03.983559 3696 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:03.983570 3696 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:03.983579 3696 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nnbcn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:05.378634 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0511 18:06:05.972372 3696 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:05.972850 3696 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:05.972880 3696 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:05.972888 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0511 18:06:05.972897 3696 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:05.972901 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0511 18:06:06.988410 3696 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:06.988972 3696 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:06.989004 3696 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:06.989011 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0511 18:06:06.989021 3696 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0511 18:06:06.989025 3696 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0511 18:06:07.000543 3696 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:07.000560 3696 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:07.000571 3696 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:07.000579 3696 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136j6whm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:08.015023 3696 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:08.015050 3696 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:08.015072 3696 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:08.015085 3696 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7jhjz6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:16.809979 3696 namespace.go:98] [cert-manager] adding namespace I0511 18:06:16.810897 3696 namespace.go:102] [cert-manager] adding namespace took 890.134µs I0511 18:06:16.850841 3696 namespace.go:142] [cert-manager] updating namespace I0511 18:06:16.861346 3696 namespace.go:142] [cert-manager] updating namespace I0511 18:06:18.414091 3696 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-5q8hn for pod on switch ip-10-0-2-22.ec2.internal I0511 18:06:18.414219 3696 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn I0511 18:06:18.428953 3696 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn] pod update took 14.753987ms I0511 18:06:18.428984 3696 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn] addLogicalPort annotation time took 14.793859ms I0511 18:06:18.430492 3696 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn] addLogicalPort took 16.419633ms, libovsdb time 956.932µs I0511 18:06:18.794277 3696 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn 1568a3364f9f53ac930ce525fc2558041e258174ffb0a0ea139e18c62405addf network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn 1568a3364f9f53ac930ce525fc2558041e258174ffb0a0ea139e18c62405addf network default NAD default NAD key ] I0511 18:06:18.803583 3696 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-5q8hn, hostIfaceName: 1568a3364f9f53a, network: default, NAD default, SandboxID: "1568a3364f9f53ac930ce525fc2558041e258174ffb0a0ea139e18c62405addf", PCI device ID: , UID: "e3ec51df-dde1-44d1-afe9-4659359bce59", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0511 18:06:18.857951 3696 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn 1568a3364f9f53ac930ce525fc2558041e258174ffb0a0ea139e18c62405addf network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-5q8hn 1568a3364f9f53ac930ce525fc2558041e258174ffb0a0ea139e18c62405addf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:a2:f9:cc:23:e4\",\"name\":\"1568a3364f9f53a\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3cdff37a-46f2-4ec9-9e3a-bd341d7fc2f9\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 18:06:25.676957 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0511 18:06:25.676977 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0511 18:06:25.677306 3696 namespace.go:142] [openshift-operators] updating namespace I0511 18:06:25.688904 3696 namespace.go:142] [openshift-operators] updating namespace W0511 18:06:27.583071 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0511 18:06:28.029762 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0511 18:06:28.077764 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0511 18:06:28.089403 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0511 18:06:28.449808 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0511 18:06:28.461779 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0511 18:06:28.485701 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0511 18:06:28.491849 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0511 18:06:31.684789 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0511 18:06:31.684807 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0511 18:06:33.051883 3696 namespace.go:142] [openshift-operators] updating namespace I0511 18:06:33.404268 3696 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-l5vgx for pod on switch ip-10-0-2-22.ec2.internal I0511 18:06:33.404397 3696 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx I0511 18:06:33.443235 3696 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx] pod update took 38.862871ms I0511 18:06:33.443262 3696 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx] addLogicalPort annotation time took 38.89259ms I0511 18:06:33.444637 3696 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx] addLogicalPort took 40.386921ms, libovsdb time 853.718µs I0511 18:06:33.765292 3696 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx 780e14607b2ff930f8043700f327cfd32c21f7312a2d63769a96ce60b21b0caf network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx 780e14607b2ff930f8043700f327cfd32c21f7312a2d63769a96ce60b21b0caf network default NAD default NAD key ] I0511 18:06:33.775228 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-l5vgx, hostIfaceName: 780e14607b2ff93, network: default, NAD default, SandboxID: "780e14607b2ff930f8043700f327cfd32c21f7312a2d63769a96ce60b21b0caf", PCI device ID: , UID: "78074d09-2259-4e10-9628-b514a91651cc", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0511 18:06:33.826141 3696 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx 780e14607b2ff930f8043700f327cfd32c21f7312a2d63769a96ce60b21b0caf network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-l5vgx 780e14607b2ff930f8043700f327cfd32c21f7312a2d63769a96ce60b21b0caf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:44:01:0c:54:8a\",\"name\":\"780e14607b2ff93\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c798e6e0-ad4a-4df7-8303-de5a3b6615e4\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:06:42.084799 3696 namespace.go:98] [trusted-artifact-signer] adding namespace I0511 18:06:42.085638 3696 namespace.go:102] [trusted-artifact-signer] adding namespace took 817.196µs W0511 18:06:42.111698 3696 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0511 18:06:42.155844 3696 namespace.go:142] [trusted-artifact-signer] updating namespace I0511 18:06:42.169798 3696 namespace.go:142] [trusted-artifact-signer] updating namespace I0511 18:06:49.679957 3696 namespace.go:98] [openshift-pipelines] adding namespace I0511 18:06:49.680892 3696 namespace.go:102] [openshift-pipelines] adding namespace took 908.697µs I0511 18:06:49.700242 3696 namespace.go:142] [openshift-pipelines] updating namespace I0511 18:06:50.391824 3696 namespace.go:142] [cert-manager] updating namespace I0511 18:06:50.594248 3696 namespace.go:142] [cert-manager-operator] updating namespace I0511 18:06:50.993961 3696 namespace.go:142] [konflux-operator] updating namespace I0511 18:06:51.196790 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:06:51.396694 3696 namespace.go:142] [open-cluster-management-2q7seai6iuch6pltn7smadh3stviopbe] updating namespace I0511 18:06:51.799404 3696 namespace.go:142] [rhbk-operator] updating namespace I0511 18:06:52.000986 3696 namespace.go:142] [rhtpa-operator] updating namespace I0511 18:06:52.197799 3696 namespace.go:142] [trusted-artifact-signer] updating namespace I0511 18:06:52.379107 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=73 I0511 18:06:52.396894 3696 namespace.go:142] [tsf] updating namespace I0511 18:06:52.600048 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:06:52.802300 3696 namespace.go:142] [tsf-tas] updating namespace I0511 18:06:53.001664 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:06:53.143099 3696 namespace.go:142] [cert-manager] updating namespace I0511 18:06:53.154926 3696 namespace.go:142] [cert-manager-operator] updating namespace I0511 18:06:53.162553 3696 namespace.go:142] [default] updating namespace I0511 18:06:53.176480 3696 namespace.go:142] [konflux-operator] updating namespace I0511 18:06:53.189120 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:06:53.204300 3696 namespace.go:142] [open-cluster-management-2q7seai6iuch6pltn7smadh3stviopbe] updating namespace I0511 18:06:53.213710 3696 namespace.go:142] [openshift] updating namespace I0511 18:06:53.228052 3696 namespace.go:142] [rhbk-operator] updating namespace I0511 18:06:53.241867 3696 namespace.go:142] [rhtpa-operator] updating namespace I0511 18:06:53.251946 3696 namespace.go:142] [trusted-artifact-signer] updating namespace I0511 18:06:53.263682 3696 namespace.go:142] [tsf] updating namespace I0511 18:06:53.277812 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:06:53.291169 3696 namespace.go:142] [tsf-tas] updating namespace I0511 18:06:53.312628 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:06:53.398528 3696 namespace.go:142] [cert-manager] updating namespace I0511 18:06:53.499215 3696 namespace.go:142] [cert-manager-operator] updating namespace I0511 18:06:53.591188 3696 namespace.go:142] [default] updating namespace I0511 18:06:53.690172 3696 namespace.go:142] [konflux-operator] updating namespace I0511 18:06:53.791459 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:06:53.890904 3696 namespace.go:142] [open-cluster-management-2q7seai6iuch6pltn7smadh3stviopbe] updating namespace I0511 18:06:53.992449 3696 namespace.go:142] [openshift] updating namespace I0511 18:06:54.091116 3696 namespace.go:142] [rhbk-operator] updating namespace I0511 18:06:54.188980 3696 namespace.go:142] [rhtpa-operator] updating namespace I0511 18:06:54.291364 3696 namespace.go:142] [trusted-artifact-signer] updating namespace I0511 18:06:54.398047 3696 namespace.go:142] [tsf] updating namespace I0511 18:06:54.494105 3696 namespace.go:142] [tsf-keycloak] updating namespace I0511 18:06:54.593349 3696 namespace.go:142] [tsf-tas] updating namespace I0511 18:06:54.691779 3696 namespace.go:142] [tsf-tpa] updating namespace I0511 18:06:55.111904 3696 namespace.go:142] [openshift-pipelines] updating namespace I0511 18:06:55.241118 3696 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:55.241670 3696 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:06:55.272179 3696 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:55.272204 3696 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:06:55.283725 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0511 18:06:55.283819 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0511 18:07:03.286836 3696 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-vc9d4 for pod on switch ip-10-0-2-22.ec2.internal I0511 18:07:03.286931 3696 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4 I0511 18:07:03.308587 3696 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4] pod update took 21.686938ms I0511 18:07:03.308610 3696 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4] addLogicalPort annotation time took 21.713923ms I0511 18:07:03.309920 3696 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4] addLogicalPort took 23.099318ms, libovsdb time 853.949µs I0511 18:07:03.653471 3696 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4 1e73a22c62c1648e83358d2b43f91067820fac3dad0837d28f31542a85df3055 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4 1e73a22c62c1648e83358d2b43f91067820fac3dad0837d28f31542a85df3055 network default NAD default NAD key ] I0511 18:07:03.662729 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-vc9d4, hostIfaceName: 1e73a22c62c1648, network: default, NAD default, SandboxID: "1e73a22c62c1648e83358d2b43f91067820fac3dad0837d28f31542a85df3055", PCI device ID: , UID: "9e227f50-1a99-480f-a7a9-68d84ed11fb1", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0511 18:07:03.714724 3696 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4 1e73a22c62c1648e83358d2b43f91067820fac3dad0837d28f31542a85df3055 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-vc9d4 1e73a22c62c1648e83358d2b43f91067820fac3dad0837d28f31542a85df3055 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:4a:d8:31:80:61\",\"name\":\"1e73a22c62c1648\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/648a38fd-338f-4623-a0c1-5d3186a612fb\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 18:07:07.773194 3696 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0511 18:07:07.773211 3696 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0511 18:07:07.773216 3696 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0511 18:07:07.889595 3696 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0511 18:07:07.889614 3696 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0511 18:07:07.889619 3696 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0511 18:07:08.207641 3696 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0511 18:07:08.207660 3696 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0511 18:07:08.207665 3696 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0511 18:07:08.207669 3696 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0511 18:07:08.467516 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 W0511 18:07:10.220068 3696 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0511 18:07:11.192597 3696 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0511 18:07:18.891463 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0511 18:07:18.891486 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0511 18:07:18.891492 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0511 18:07:26.290956 3696 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-nfk2q for pod on switch ip-10-0-2-22.ec2.internal I0511 18:07:26.291063 3696 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q I0511 18:07:26.306890 3696 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q] pod update took 15.853058ms I0511 18:07:26.306919 3696 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q] addLogicalPort annotation time took 15.887766ms I0511 18:07:26.308216 3696 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q] addLogicalPort took 17.274741ms, libovsdb time 884.593µs I0511 18:07:26.650964 3696 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q a24417c7672853dcd9f8d9128d06f060a0333450f5814f515cbf331ef53a845a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q a24417c7672853dcd9f8d9128d06f060a0333450f5814f515cbf331ef53a845a network default NAD default NAD key ] I0511 18:07:26.660952 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-nfk2q, hostIfaceName: a24417c7672853d, network: default, NAD default, SandboxID: "a24417c7672853dcd9f8d9128d06f060a0333450f5814f515cbf331ef53a845a", PCI device ID: , UID: "734321b2-83bc-4119-bed8-9e42832275b5", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0511 18:07:26.714038 3696 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q a24417c7672853dcd9f8d9128d06f060a0333450f5814f515cbf331ef53a845a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-nfk2q a24417c7672853dcd9f8d9128d06f060a0333450f5814f515cbf331ef53a845a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:e3:5a:60:1d:ef\",\"name\":\"a24417c7672853d\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/64d265ec-a158-4d30-8753-ea6857a26a6f\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 18:07:31.000342 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0511 18:07:35.448859 3696 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:07:35.449394 3696 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:07:35.468825 3696 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:07:35.468843 3696 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:07:35.473996 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0511 18:07:35.474017 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0511 18:07:37.590673 3696 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0511 18:07:37.591342 3696 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0511 18:07:37.591368 3696 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0511 18:07:37.593475 3696 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0511 18:07:42.014771 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0511 18:07:43.874530 3696 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-s8rr7 for pod on switch ip-10-0-2-22.ec2.internal I0511 18:07:43.874645 3696 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7 I0511 18:07:43.888794 3696 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7] pod update took 14.184402ms I0511 18:07:43.888817 3696 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7] addLogicalPort annotation time took 14.213599ms I0511 18:07:43.890102 3696 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7] addLogicalPort took 15.610121ms, libovsdb time 874.154µs I0511 18:07:44.233936 3696 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7 e911d1cc4576be696ce8af19c4915766e3e1741903a91d6c57190df9ddf0bb3b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7 e911d1cc4576be696ce8af19c4915766e3e1741903a91d6c57190df9ddf0bb3b network default NAD default NAD key ] I0511 18:07:44.244470 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-s8rr7, hostIfaceName: e911d1cc4576be6, network: default, NAD default, SandboxID: "e911d1cc4576be696ce8af19c4915766e3e1741903a91d6c57190df9ddf0bb3b", PCI device ID: , UID: "96d0de43-4fc7-4280-aca1-dd57a8d5d49f", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0511 18:07:44.296472 3696 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7 e911d1cc4576be696ce8af19c4915766e3e1741903a91d6c57190df9ddf0bb3b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-s8rr7 e911d1cc4576be696ce8af19c4915766e3e1741903a91d6c57190df9ddf0bb3b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:b7:3d:26:4e:0a\",\"name\":\"e911d1cc4576be6\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/815d00eb-b381-4a36-8fbc-03558bbf1feb\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 18:07:46.380967 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0511 18:07:46.380984 3696 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0511 18:07:46.381099 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0511 18:07:46.381114 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0511 18:07:47.749487 3696 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0511 18:07:50.827134 3696 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-22.ec2.internal I0511 18:07:50.827245 3696 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0511 18:07:50.840676 3696 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 13.457117ms I0511 18:07:50.840701 3696 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 13.491942ms I0511 18:07:50.842056 3696 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 14.942654ms, libovsdb time 890.961µs I0511 18:07:55.689417 3696 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 0e1bf957614ee0f4e60f3b7e1494d314bac5d9dcfddb8bb6945e880b4ada5188 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 0e1bf957614ee0f4e60f3b7e1494d314bac5d9dcfddb8bb6945e880b4ada5188 network default NAD default NAD key ] I0511 18:07:55.699759 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 0e1bf957614ee0f, network: default, NAD default, SandboxID: "0e1bf957614ee0f4e60f3b7e1494d314bac5d9dcfddb8bb6945e880b4ada5188", PCI device ID: , UID: "43b901a3-97d0-47d9-9654-884bbc76f513", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0511 18:07:55.754935 3696 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 0e1bf957614ee0f4e60f3b7e1494d314bac5d9dcfddb8bb6945e880b4ada5188 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 0e1bf957614ee0f4e60f3b7e1494d314bac5d9dcfddb8bb6945e880b4ada5188 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:f1:d5:13:e6:15\",\"name\":\"0e1bf957614ee0f\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de55e66c-1ff9-4c5f-bc81-ef7572a8e395\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:08:03.219435 3696 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0511 18:08:04.655523 3696 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-5ztnb for pod on switch ip-10-0-2-22.ec2.internal I0511 18:08:04.655648 3696 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb I0511 18:08:04.669397 3696 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb] pod update took 13.767624ms I0511 18:08:04.669420 3696 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb] addLogicalPort annotation time took 13.796346ms I0511 18:08:04.670789 3696 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb] addLogicalPort took 15.30547ms, libovsdb time 932.962µs I0511 18:08:04.762189 3696 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-fgltp for pod on switch ip-10-0-2-22.ec2.internal I0511 18:08:04.762349 3696 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp I0511 18:08:04.776889 3696 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp] pod update took 14.57466ms I0511 18:08:04.776912 3696 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp] addLogicalPort annotation time took 14.600302ms I0511 18:08:04.778189 3696 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp] addLogicalPort took 16.021161ms, libovsdb time 840.686µs I0511 18:08:05.016962 3696 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb a98ed357561d186a005663d4e956cb85bd84f73d6badab4cccf5a953c1cb5333 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb a98ed357561d186a005663d4e956cb85bd84f73d6badab4cccf5a953c1cb5333 network default NAD default NAD key ] I0511 18:08:05.027008 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-5ztnb, hostIfaceName: a98ed357561d186, network: default, NAD default, SandboxID: "a98ed357561d186a005663d4e956cb85bd84f73d6badab4cccf5a953c1cb5333", PCI device ID: , UID: "91feae34-528f-476e-9412-4c19bad9372e", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0511 18:08:05.080577 3696 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb a98ed357561d186a005663d4e956cb85bd84f73d6badab4cccf5a953c1cb5333 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-5ztnb a98ed357561d186a005663d4e956cb85bd84f73d6badab4cccf5a953c1cb5333 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:2e:dd:61:21:ff\",\"name\":\"a98ed357561d186\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5ed45a4f-8fd9-4bb5-9c86-c782404542ad\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:08:05.124310 3696 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp e4dd9c22152a0840f627374b634c28477d5f3552a0070a2cfa74bac4832c3270 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp e4dd9c22152a0840f627374b634c28477d5f3552a0070a2cfa74bac4832c3270 network default NAD default NAD key ] I0511 18:08:05.134297 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-fgltp, hostIfaceName: e4dd9c22152a084, network: default, NAD default, SandboxID: "e4dd9c22152a0840f627374b634c28477d5f3552a0070a2cfa74bac4832c3270", PCI device ID: , UID: "997b70fe-1d9a-4fb8-a81d-e663316a8e00", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0511 18:08:05.190145 3696 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp e4dd9c22152a0840f627374b634c28477d5f3552a0070a2cfa74bac4832c3270 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-fgltp e4dd9c22152a0840f627374b634c28477d5f3552a0070a2cfa74bac4832c3270 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:b5:3e:a5:af:e9\",\"name\":\"e4dd9c22152a084\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05ff02be-d882-4556-8b1b-01c21daed9e5\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:08:06.878664 3696 namespace.go:142] [openshift-pipelines] updating namespace W0511 18:08:10.512185 3696 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0511 18:08:10.512204 3696 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0511 18:08:10.512208 3696 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0511 18:08:14.493524 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0511 18:08:27.739593 3696 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:08:27.740120 3696 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:08:27.780021 3696 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:08:27.780046 3696 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:08:27.783058 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0511 18:08:27.783085 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0511 18:08:30.678390 3696 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-74fd986b89-7xrl6] creating logical port tsf-tas_trillian-logserver-74fd986b89-7xrl6 for pod on switch ip-10-0-2-22.ec2.internal I0511 18:08:30.678485 3696 kube.go:256] Updating pod tsf-tas/trillian-logserver-74fd986b89-7xrl6 I0511 18:08:30.699364 3696 pod.go:62] [tsf-tas/trillian-logserver-74fd986b89-7xrl6] pod update took 20.906673ms I0511 18:08:30.699383 3696 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-74fd986b89-7xrl6] addLogicalPort annotation time took 20.93053ms I0511 18:08:30.700851 3696 pods.go:271] [tsf-tas/trillian-logserver-74fd986b89-7xrl6] addLogicalPort took 22.477161ms, libovsdb time 984.71µs I0511 18:08:30.808541 3696 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-vn5mc-2dmc4] creating logical port tsf-tas_rekor-createtree-job-vn5mc-2dmc4 for pod on switch ip-10-0-2-22.ec2.internal I0511 18:08:30.808693 3696 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-vn5mc-2dmc4 I0511 18:08:30.824396 3696 pod.go:62] [tsf-tas/rekor-createtree-job-vn5mc-2dmc4] pod update took 15.734515ms I0511 18:08:30.824418 3696 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-vn5mc-2dmc4] addLogicalPort annotation time took 15.761873ms I0511 18:08:30.825856 3696 pods.go:271] [tsf-tas/rekor-createtree-job-vn5mc-2dmc4] addLogicalPort took 17.333919ms, libovsdb time 996.157µs I0511 18:08:31.170181 3696 cni.go:452] [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key ] I0511 18:08:31.180142 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-vn5mc-2dmc4, hostIfaceName: aae32e0add18363, network: default, NAD default, SandboxID: "aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f", PCI device ID: , UID: "d7bccf08-4ec0-4905-af2a-8fd433fbca50", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0511 18:08:31.235098 3696 cni.go:473] [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:ed:73:eb:ce:9c\",\"name\":\"aae32e0add18363\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bcdc164e-0941-4fad-aa73-43f8c3d2ff58\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:08:31.641243 3696 cni.go:452] [tsf-tas/trillian-logserver-74fd986b89-7xrl6 68aeea97eb248e6d026391d2d5a1d7ccea20bb63e93cd3e03836fa8a6191586d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-74fd986b89-7xrl6 68aeea97eb248e6d026391d2d5a1d7ccea20bb63e93cd3e03836fa8a6191586d network default NAD default NAD key ] I0511 18:08:31.651153 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-74fd986b89-7xrl6, hostIfaceName: 68aeea97eb248e6, network: default, NAD default, SandboxID: "68aeea97eb248e6d026391d2d5a1d7ccea20bb63e93cd3e03836fa8a6191586d", PCI device ID: , UID: "dabae4eb-3fc1-4119-9760-dd1924d9612e", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0511 18:08:31.703130 3696 cni.go:473] [tsf-tas/trillian-logserver-74fd986b89-7xrl6 68aeea97eb248e6d026391d2d5a1d7ccea20bb63e93cd3e03836fa8a6191586d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-74fd986b89-7xrl6 68aeea97eb248e6d026391d2d5a1d7ccea20bb63e93cd3e03836fa8a6191586d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:68:f3:00:e2:a1\",\"name\":\"68aeea97eb248e6\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3355cd70-1f47-4f40-b54c-78d1469892c3\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 18:08:34.211751 3696 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0511 18:08:34.211775 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0511 18:08:39.404867 3696 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0511 18:08:39.680910 3696 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-jfqq8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:08:39.681520 3696 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-jfqq8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0511 18:08:40.688721 3696 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0511 18:08:40.706261 3696 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-jfqq8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:08:40.706285 3696 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-jfqq8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:08:41.466125 3696 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0511 18:08:49.054913 3696 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-f5d4g-vzvtz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:08:49.055489 3696 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-f5d4g-vzvtz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:08:49.302244 3696 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-f5d4g-vzvtz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:08:49.302279 3696 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-f5d4g-vzvtz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0511 18:08:49.313711 3696 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0511 18:08:53.757462 3696 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0511 18:08:54.764240 3696 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0511 18:08:54.764259 3696 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0511 18:09:17.544255 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0511 18:09:21.035577 3696 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:21.036092 3696 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:21.054234 3696 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:21.054253 3696 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:21.061132 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0511 18:09:21.061154 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0511 18:09:22.620141 3696 cni.go:452] [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key ] I0511 18:09:22.713200 3696 cni.go:473] [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-vn5mc-2dmc4 aae32e0add183631831069f9effbdb0b93c26f139233036554c0f4d3f6a5975f network default NAD default NAD key default], result "{}", err I0511 18:09:22.764979 3696 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-vn5mc-2dmc4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:22.765007 3696 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-vn5mc-2dmc4 I0511 18:09:22.766341 3696 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-vn5mc-2dmc4, ips: 10.128.0.47 I0511 18:09:22.766369 3696 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-vn5mc-2dmc4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:23.636058 3696 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-vn5mc-2dmc4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:23.636082 3696 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-vn5mc-2dmc4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:26.877297 3696 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7] creating logical port tsf-tas_rekor-search-ui-68bd7fcb59-xzjk7 for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:26.877380 3696 kube.go:256] Updating pod tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7 I0511 18:09:26.890107 3696 pod.go:62] [tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7] pod update took 12.747047ms I0511 18:09:26.890133 3696 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7] addLogicalPort annotation time took 12.775841ms I0511 18:09:26.891632 3696 pods.go:271] [tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7] addLogicalPort took 14.354933ms, libovsdb time 895.676µs I0511 18:09:27.235758 3696 cni.go:452] [tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7 3ae9680b940ad56f8bdc500c724bdc6a0b204f564d4b6e70875ec97ce763a255 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7 3ae9680b940ad56f8bdc500c724bdc6a0b204f564d4b6e70875ec97ce763a255 network default NAD default NAD key ] I0511 18:09:27.245105 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-68bd7fcb59-xzjk7, hostIfaceName: 3ae9680b940ad56, network: default, NAD default, SandboxID: "3ae9680b940ad56f8bdc500c724bdc6a0b204f564d4b6e70875ec97ce763a255", PCI device ID: , UID: "b9ea0fd1-e315-48d8-8e6f-05f7c4590380", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0511 18:09:27.298478 3696 cni.go:473] [tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7 3ae9680b940ad56f8bdc500c724bdc6a0b204f564d4b6e70875ec97ce763a255 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-68bd7fcb59-xzjk7 3ae9680b940ad56f8bdc500c724bdc6a0b204f564d4b6e70875ec97ce763a255 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:fa:67:0a:e0:bb\",\"name\":\"3ae9680b940ad56\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/969a1d14-43ce-400d-ae57-ccc285ac9170\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:29.462761 3696 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-848fb4549f-gj97g] creating logical port tsf-tas_rekor-server-848fb4549f-gj97g for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:29.462862 3696 kube.go:256] Updating pod tsf-tas/rekor-server-848fb4549f-gj97g I0511 18:09:29.477006 3696 pod.go:62] [tsf-tas/rekor-server-848fb4549f-gj97g] pod update took 14.169436ms I0511 18:09:29.477034 3696 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-848fb4549f-gj97g] addLogicalPort annotation time took 14.201599ms I0511 18:09:29.478812 3696 pods.go:271] [tsf-tas/rekor-server-848fb4549f-gj97g] addLogicalPort took 16.069268ms, libovsdb time 1.104011ms W0511 18:09:29.503052 3696 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0511 18:09:30.094992 3696 obj_retry.go:498] Detected object tsf/patch-tekton-config-cwh9d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:30.095605 3696 obj_retry.go:498] Detected object tsf/patch-tekton-config-cwh9d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0511 18:09:30.671397 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0511 18:09:30.972880 3696 obj_retry.go:492] Detected object tsf/patch-tekton-config-cwh9d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:30.972903 3696 obj_retry.go:492] Detected object tsf/patch-tekton-config-cwh9d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:31.073680 3696 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-g4g6k] creating logical port tsf_tsf-tekton-configuration-g4g6k for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:31.073790 3696 kube.go:256] Updating pod tsf/tsf-tekton-configuration-g4g6k I0511 18:09:31.088330 3696 pod.go:62] [tsf/tsf-tekton-configuration-g4g6k] pod update took 14.571606ms I0511 18:09:31.088355 3696 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-g4g6k] addLogicalPort annotation time took 14.60351ms I0511 18:09:31.089645 3696 pods.go:271] [tsf/tsf-tekton-configuration-g4g6k] addLogicalPort took 15.987009ms, libovsdb time 806.808µs I0511 18:09:31.435291 3696 cni.go:452] [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key ] I0511 18:09:31.446116 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-g4g6k, hostIfaceName: cc9121454adf3ed, network: default, NAD default, SandboxID: "cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c", PCI device ID: , UID: "f89dc989-636b-400d-b589-9502eae1367a", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0511 18:09:31.510428 3696 cni.go:473] [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:6b:4f:86:3b:a5\",\"name\":\"cc9121454adf3ed\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fa215088-e173-4a2c-8a6e-20592afb0346\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:34.632438 3696 cni.go:452] [tsf-tas/rekor-server-848fb4549f-gj97g c79f6f3ba0c64f4f4093ec7a26290ffb3d0b8102fa6979c05cb2c84158a4e3c6 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-848fb4549f-gj97g c79f6f3ba0c64f4f4093ec7a26290ffb3d0b8102fa6979c05cb2c84158a4e3c6 network default NAD default NAD key ] I0511 18:09:34.641988 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-848fb4549f-gj97g, hostIfaceName: c79f6f3ba0c64f4, network: default, NAD default, SandboxID: "c79f6f3ba0c64f4f4093ec7a26290ffb3d0b8102fa6979c05cb2c84158a4e3c6", PCI device ID: , UID: "2786264d-cf11-4fe5-a0c3-c36a3653cc98", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0511 18:09:34.696629 3696 cni.go:473] [tsf-tas/rekor-server-848fb4549f-gj97g c79f6f3ba0c64f4f4093ec7a26290ffb3d0b8102fa6979c05cb2c84158a4e3c6 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-848fb4549f-gj97g c79f6f3ba0c64f4f4093ec7a26290ffb3d0b8102fa6979c05cb2c84158a4e3c6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:2f:1f:bc:39:03\",\"name\":\"c79f6f3ba0c64f4\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d88c1de2-a934-422b-a8be-ed2816af6443\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:35.708093 3696 cni.go:452] [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key ] I0511 18:09:35.778110 3696 cni.go:473] [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-g4g6k cc9121454adf3edbdd39fbb16ca224b627f60473447b223c0e7af2bcc59a197c network default NAD default NAD key default], result "{}", err I0511 18:09:35.832452 3696 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-g4g6k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:35.832514 3696 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-g4g6k I0511 18:09:35.833872 3696 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-g4g6k, ips: 10.128.0.50 I0511 18:09:35.833896 3696 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-g4g6k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:36.854153 3696 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-g4g6k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:36.854189 3696 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-g4g6k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:36.888314 3696 obj_retry.go:492] Detected object tsf/patch-tekton-config-cwh9d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:36.888341 3696 obj_retry.go:492] Detected object tsf/patch-tekton-config-cwh9d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:36.892776 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-cwh9d I0511 18:09:36.892806 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-cwh9d W0511 18:09:37.722093 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0511 18:09:42.147523 3696 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:42.148134 3696 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:42.168226 3696 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:42.168250 3696 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:42.173981 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0511 18:09:42.174013 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0511 18:09:42.678517 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0511 18:09:43.482999 3696 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-qmtcr] creating logical port tsf-tpa_create-db-qmtcr for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:43.483090 3696 kube.go:256] Updating pod tsf-tpa/create-db-qmtcr I0511 18:09:43.499662 3696 pod.go:62] [tsf-tpa/create-db-qmtcr] pod update took 16.597175ms I0511 18:09:43.499683 3696 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-qmtcr] addLogicalPort annotation time took 16.624119ms I0511 18:09:43.500942 3696 pods.go:271] [tsf-tpa/create-db-qmtcr] addLogicalPort took 17.960223ms, libovsdb time 848.014µs I0511 18:09:43.846778 3696 cni.go:452] [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key ] I0511 18:09:43.856742 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-qmtcr, hostIfaceName: 09e78648d800f4f, network: default, NAD default, SandboxID: "09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e", PCI device ID: , UID: "b06f4e8d-84de-4439-92e1-3bed7f4d5f20", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0511 18:09:43.910336 3696 cni.go:473] [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:ea:13:0e:43:cd\",\"name\":\"09e78648d800f4f\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67df9f85-0b83-4521-a324-dccc1d3ebe89\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:47.500171 3696 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0511 18:09:47.779979 3696 cni.go:452] [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key ] I0511 18:09:47.868288 3696 cni.go:473] [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-qmtcr 09e78648d800f4ffb00aa68fdae6ac331d92a4acd2ac637e4da105b3ba8fb97e network default NAD default NAD key default], result "{}", err I0511 18:09:47.920060 3696 obj_retry.go:498] Detected object tsf-tpa/create-db-qmtcr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:47.920095 3696 pods.go:176] Deleting pod: tsf-tpa/create-db-qmtcr I0511 18:09:47.921538 3696 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-qmtcr, ips: 10.128.0.51 I0511 18:09:47.921565 3696 obj_retry.go:498] Detected object tsf-tpa/create-db-qmtcr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0511 18:09:48.730800 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0511 18:09:48.952321 3696 obj_retry.go:492] Detected object tsf-tpa/create-db-qmtcr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:48.952348 3696 obj_retry.go:492] Detected object tsf-tpa/create-db-qmtcr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:49.049447 3696 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-klwgs] creating logical port tsf-tpa_migrate-db-klwgs for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:49.049598 3696 kube.go:256] Updating pod tsf-tpa/migrate-db-klwgs I0511 18:09:49.062213 3696 pod.go:62] [tsf-tpa/migrate-db-klwgs] pod update took 12.635554ms I0511 18:09:49.062238 3696 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-klwgs] addLogicalPort annotation time took 12.666793ms I0511 18:09:49.063631 3696 pods.go:271] [tsf-tpa/migrate-db-klwgs] addLogicalPort took 14.201946ms, libovsdb time 898.236µs I0511 18:09:49.408161 3696 cni.go:452] [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key ] I0511 18:09:49.417718 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-klwgs, hostIfaceName: 072eb45a0e2544a, network: default, NAD default, SandboxID: "072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4", PCI device ID: , UID: "7fa3c6bd-855d-4539-8b59-eb95d7094dca", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0511 18:09:49.470976 3696 cni.go:473] [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:d8:77:35:6a:a4\",\"name\":\"072eb45a0e2544a\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0befafb-7cf4-43ec-bf24-ffcde20a577f\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:49.734269 3696 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-nm64k-7lpxs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:49.734870 3696 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-nm64k-7lpxs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:50.626626 3696 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-nm64k-7lpxs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:50.626650 3696 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-nm64k-7lpxs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:51.805857 3696 cni.go:452] [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key ] I0511 18:09:51.887895 3696 cni.go:473] [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-klwgs 072eb45a0e2544aececfccfa988f00ef875c42d124c1f88f8162c433cdc441b4 network default NAD default NAD key default], result "{}", err I0511 18:09:51.938899 3696 obj_retry.go:498] Detected object tsf-tpa/migrate-db-klwgs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:51.938932 3696 pods.go:176] Deleting pod: tsf-tpa/migrate-db-klwgs I0511 18:09:51.940455 3696 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-klwgs, ips: 10.128.0.52 I0511 18:09:51.940482 3696 obj_retry.go:498] Detected object tsf-tpa/migrate-db-klwgs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:52.083586 3696 obj_retry.go:492] Detected object tsf-tpa/migrate-db-klwgs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:52.083608 3696 obj_retry.go:492] Detected object tsf-tpa/migrate-db-klwgs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:52.263860 3696 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-kc88q] creating logical port tsf-tpa_create-importers-kc88q for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:52.263976 3696 kube.go:256] Updating pod tsf-tpa/create-importers-kc88q I0511 18:09:52.280078 3696 pod.go:62] [tsf-tpa/create-importers-kc88q] pod update took 16.121557ms I0511 18:09:52.280101 3696 base_network_controller_pods.go:951] [default/tsf-tpa/create-importers-kc88q] addLogicalPort annotation time took 16.14852ms I0511 18:09:52.281403 3696 pods.go:271] [tsf-tpa/create-importers-kc88q] addLogicalPort took 17.562506ms, libovsdb time 849.924µs I0511 18:09:52.924360 3696 cni.go:452] [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key ] I0511 18:09:52.934418 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-kc88q, hostIfaceName: 996488373c47e04, network: default, NAD default, SandboxID: "996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964", PCI device ID: , UID: "2cd73800-0939-4e80-be8e-5a2b398dd2da", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0511 18:09:52.984951 3696 cni.go:473] [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:97:b8:af:7c:58\",\"name\":\"996488373c47e04\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9be4c46c-2833-4d61-9cd3-b17edbf565dc\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:54.826599 3696 cni.go:452] [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key ] I0511 18:09:54.912950 3696 cni.go:473] [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-importers-kc88q 996488373c47e0422dc0114eba332e405d5503de61ac8782fc122e0737ce8964 network default NAD default NAD key default], result "{}", err I0511 18:09:54.970185 3696 obj_retry.go:498] Detected object tsf-tpa/create-importers-kc88q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:54.970216 3696 pods.go:176] Deleting pod: tsf-tpa/create-importers-kc88q I0511 18:09:54.971601 3696 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-kc88q, ips: 10.128.0.53 I0511 18:09:54.971626 3696 obj_retry.go:498] Detected object tsf-tpa/create-importers-kc88q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:09:55.718342 3696 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-c74d7d459-p6rfz] creating logical port tsf-tas_ctlog-c74d7d459-p6rfz for pod on switch ip-10-0-2-22.ec2.internal I0511 18:09:55.718466 3696 kube.go:256] Updating pod tsf-tas/ctlog-c74d7d459-p6rfz I0511 18:09:55.743381 3696 pod.go:62] [tsf-tas/ctlog-c74d7d459-p6rfz] pod update took 24.936869ms I0511 18:09:55.743403 3696 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-c74d7d459-p6rfz] addLogicalPort annotation time took 24.962791ms I0511 18:09:55.744947 3696 pods.go:271] [tsf-tas/ctlog-c74d7d459-p6rfz] addLogicalPort took 26.620968ms, libovsdb time 1.019453ms I0511 18:09:56.006463 3696 obj_retry.go:492] Detected object tsf-tpa/create-importers-kc88q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:56.006484 3696 obj_retry.go:492] Detected object tsf-tpa/create-importers-kc88q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:09:56.079509 3696 cni.go:452] [tsf-tas/ctlog-c74d7d459-p6rfz 31e3ec637619e12c7512b9f544eb08c62f6fd207d80f9af1a35de101ce306101 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-c74d7d459-p6rfz 31e3ec637619e12c7512b9f544eb08c62f6fd207d80f9af1a35de101ce306101 network default NAD default NAD key ] I0511 18:09:56.090072 3696 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-c74d7d459-p6rfz, hostIfaceName: 31e3ec637619e12, network: default, NAD default, SandboxID: "31e3ec637619e12c7512b9f544eb08c62f6fd207d80f9af1a35de101ce306101", PCI device ID: , UID: "646f06e1-ea5a-4bad-9688-a6007534b14e", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0511 18:09:56.144059 3696 cni.go:473] [tsf-tas/ctlog-c74d7d459-p6rfz 31e3ec637619e12c7512b9f544eb08c62f6fd207d80f9af1a35de101ce306101 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-c74d7d459-p6rfz 31e3ec637619e12c7512b9f544eb08c62f6fd207d80f9af1a35de101ce306101 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:5e:61:95:8c:b3\",\"name\":\"31e3ec637619e12\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d2a9f6d5-d83f-4f91-b055-67530919a4ae\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:09:57.376472 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=113 W0511 18:09:59.856165 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0511 18:10:00.175764 3696 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29642050-6qx8g for pod on switch ip-10-0-2-22.ec2.internal I0511 18:10:00.175860 3696 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g I0511 18:10:00.190672 3696 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g] pod update took 14.836155ms I0511 18:10:00.190710 3696 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g] addLogicalPort annotation time took 14.880263ms I0511 18:10:00.192330 3696 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g] addLogicalPort took 16.582111ms, libovsdb time 980.67µs I0511 18:10:00.534710 3696 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key ] I0511 18:10:00.544049 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29642050-6qx8g, hostIfaceName: d25e9e110c3397c, network: default, NAD default, SandboxID: "d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823", PCI device ID: , UID: "2cbacaf0-1125-4ca8-9b53-66f2ed576054", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0511 18:10:00.598626 3696 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:31:d9:3f:2d:2b\",\"name\":\"d25e9e110c3397c\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13885e99-e459-4187-8589-ce7ccccfb3f1\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:10:04.889334 3696 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key ] I0511 18:10:04.979650 3696 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g d25e9e110c3397c1cdad2f8de6236e4d70186fe3fc8083bd40b067bcd73a4823 network default NAD default NAD key default], result "{}", err I0511 18:10:05.163348 3696 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:05.163385 3696 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g I0511 18:10:05.165060 3696 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g, ips: 10.128.0.55 I0511 18:10:05.165089 3696 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:05.904027 3696 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:05.904052 3696 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642050-6qx8g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0511 18:10:07.191380 3696 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0511 18:10:07.848912 3696 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-492nt-cknwd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:07.849465 3696 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-492nt-cknwd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:08.870884 3696 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-492nt-cknwd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:08.870906 3696 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-492nt-cknwd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:10.332653 3696 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:10.333188 3696 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:10.353097 3696 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:10.353119 3696 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:10.359508 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0511 18:10:10.359534 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa W0511 18:10:10.865777 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0511 18:10:11.221315 3696 namespace.go:98] [integration-service] adding namespace I0511 18:10:11.222260 3696 namespace.go:102] [integration-service] adding namespace took 919.909µs I0511 18:10:11.224454 3696 namespace.go:98] [image-controller] adding namespace I0511 18:10:11.224552 3696 namespace.go:98] [build-service] adding namespace I0511 18:10:11.225154 3696 namespace.go:102] [image-controller] adding namespace took 682.025µs I0511 18:10:11.225787 3696 namespace.go:102] [build-service] adding namespace took 1.218241ms I0511 18:10:11.226416 3696 namespace.go:98] [default-tenant] adding namespace I0511 18:10:11.226966 3696 namespace.go:102] [default-tenant] adding namespace took 538.07µs I0511 18:10:11.270208 3696 namespace.go:142] [integration-service] updating namespace I0511 18:10:11.327850 3696 namespace.go:142] [integration-service] updating namespace I0511 18:10:11.328380 3696 namespace.go:142] [image-controller] updating namespace I0511 18:10:11.371820 3696 namespace.go:142] [image-controller] updating namespace I0511 18:10:11.409929 3696 namespace.go:142] [build-service] updating namespace I0511 18:10:11.460934 3696 namespace.go:142] [build-service] updating namespace I0511 18:10:11.490095 3696 namespace.go:142] [default-tenant] updating namespace I0511 18:10:11.532439 3696 namespace.go:142] [default-tenant] updating namespace I0511 18:10:11.596390 3696 namespace.go:142] [build-service] updating namespace I0511 18:10:11.626004 3696 namespace.go:142] [integration-service] updating namespace I0511 18:10:11.684708 3696 namespace.go:98] [release-service] adding namespace I0511 18:10:11.685604 3696 namespace.go:102] [release-service] adding namespace took 874.436µs I0511 18:10:11.737035 3696 namespace.go:142] [release-service] updating namespace I0511 18:10:11.776619 3696 namespace.go:142] [release-service] updating namespace I0511 18:10:11.840043 3696 namespace.go:142] [konflux-ui] updating namespace I0511 18:10:12.027953 3696 namespace.go:98] [konflux-info] adding namespace I0511 18:10:12.028861 3696 namespace.go:102] [konflux-info] adding namespace took 887.299µs I0511 18:10:12.104030 3696 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-22.ec2.internal I0511 18:10:12.104043 3696 namespace.go:142] [konflux-info] updating namespace I0511 18:10:12.104148 3696 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0511 18:10:12.104165 3696 namespace.go:98] [namespace-lister] adding namespace I0511 18:10:12.105015 3696 namespace.go:102] [namespace-lister] adding namespace took 836.256µs I0511 18:10:12.136459 3696 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 32.345265ms I0511 18:10:12.136481 3696 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 32.374321ms I0511 18:10:12.137813 3696 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 33.804356ms, libovsdb time 832.263µs I0511 18:10:12.241432 3696 namespace.go:98] [enterprise-contract-service] adding namespace I0511 18:10:12.242309 3696 namespace.go:102] [enterprise-contract-service] adding namespace took 857.583µs I0511 18:10:12.242354 3696 namespace.go:142] [namespace-lister] updating namespace I0511 18:10:12.252914 3696 namespace.go:142] [konflux-info] updating namespace I0511 18:10:12.344141 3696 namespace.go:142] [image-controller] updating namespace I0511 18:10:12.356991 3696 namespace.go:142] [namespace-lister] updating namespace I0511 18:10:12.415979 3696 namespace.go:142] [enterprise-contract-service] updating namespace I0511 18:10:12.492166 3696 namespace.go:142] [enterprise-contract-service] updating namespace I0511 18:10:12.602765 3696 namespace.go:142] [default-tenant] updating namespace I0511 18:10:12.681150 3696 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0511 18:10:12.681793 3696 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0511 18:10:12.681808 3696 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0511 18:10:12.684401 3696 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0511 18:10:12.737889 3696 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0511 18:10:12.737914 3696 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0511 18:10:12.738757 3696 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0511 18:10:12.771739 3696 namespace.go:98] [segment-bridge] adding namespace I0511 18:10:12.772605 3696 namespace.go:102] [segment-bridge] adding namespace took 844.951µs I0511 18:10:12.828848 3696 namespace.go:98] [konflux-cli] adding namespace I0511 18:10:12.829665 3696 namespace.go:102] [konflux-cli] adding namespace took 797.477µs I0511 18:10:12.901354 3696 namespace.go:142] [segment-bridge] updating namespace I0511 18:10:13.026155 3696 namespace.go:142] [segment-bridge] updating namespace I0511 18:10:13.590883 3696 namespace.go:142] [konflux-cli] updating namespace I0511 18:10:13.680693 3696 cni.go:452] [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key ] I0511 18:10:13.690789 3696 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: 9b8a421142e46f6, network: default, NAD default, SandboxID: "9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33", PCI device ID: , UID: "24f74901-11d6-4ef3-a304-bb9a1f8ce1ff", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0511 18:10:13.755016 3696 cni.go:473] [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:a1:7e:43:1d:af\",\"name\":\"9b8a421142e46f6\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb8683e7-00b0-4f49-958d-d2fe7db45a33\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:10:14.066858 3696 namespace.go:142] [build-service] updating namespace I0511 18:10:14.193507 3696 namespace.go:142] [konflux-cli] updating namespace I0511 18:10:15.025479 3696 base_network_controller_pods.go:487] [default/konflux-ui/proxy-6557cd967f-pdhwm] creating logical port konflux-ui_proxy-6557cd967f-pdhwm for pod on switch ip-10-0-2-22.ec2.internal I0511 18:10:15.025600 3696 kube.go:256] Updating pod konflux-ui/proxy-6557cd967f-pdhwm I0511 18:10:15.065335 3696 pod.go:62] [konflux-ui/proxy-6557cd967f-pdhwm] pod update took 39.772787ms I0511 18:10:15.065359 3696 base_network_controller_pods.go:951] [default/konflux-ui/proxy-6557cd967f-pdhwm] addLogicalPort annotation time took 39.80335ms I0511 18:10:15.066914 3696 pods.go:271] [konflux-ui/proxy-6557cd967f-pdhwm] addLogicalPort took 41.451733ms, libovsdb time 1.054961ms I0511 18:10:15.363795 3696 namespace.go:142] [default-tenant] updating namespace I0511 18:10:15.716447 3696 namespace.go:142] [enterprise-contract-service] updating namespace I0511 18:10:15.902814 3696 namespace.go:142] [image-controller] updating namespace I0511 18:10:16.194096 3696 namespace.go:142] [integration-service] updating namespace I0511 18:10:16.505289 3696 namespace.go:142] [konflux-cli] updating namespace I0511 18:10:16.763148 3696 namespace.go:142] [konflux-info] updating namespace I0511 18:10:16.981176 3696 namespace.go:142] [namespace-lister] updating namespace I0511 18:10:17.169537 3696 namespace.go:142] [release-service] updating namespace I0511 18:10:17.385650 3696 namespace.go:142] [segment-bridge] updating namespace I0511 18:10:17.500421 3696 namespace.go:142] [build-service] updating namespace I0511 18:10:17.524208 3696 namespace.go:142] [default-tenant] updating namespace I0511 18:10:17.542085 3696 namespace.go:142] [enterprise-contract-service] updating namespace I0511 18:10:17.555771 3696 namespace.go:142] [image-controller] updating namespace I0511 18:10:17.572927 3696 namespace.go:142] [integration-service] updating namespace I0511 18:10:17.588808 3696 namespace.go:142] [konflux-cli] updating namespace I0511 18:10:17.616338 3696 namespace.go:142] [konflux-info] updating namespace I0511 18:10:17.620958 3696 namespace.go:142] [namespace-lister] updating namespace I0511 18:10:17.632753 3696 namespace.go:142] [release-service] updating namespace I0511 18:10:17.645860 3696 namespace.go:142] [segment-bridge] updating namespace I0511 18:10:17.697915 3696 namespace.go:142] [build-service] updating namespace I0511 18:10:17.751923 3696 namespace.go:142] [default-tenant] updating namespace W0511 18:10:17.803836 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0511 18:10:17.837923 3696 namespace.go:142] [enterprise-contract-service] updating namespace I0511 18:10:17.932930 3696 namespace.go:142] [image-controller] updating namespace I0511 18:10:18.031911 3696 namespace.go:142] [integration-service] updating namespace I0511 18:10:18.135068 3696 namespace.go:142] [konflux-cli] updating namespace I0511 18:10:18.237981 3696 namespace.go:142] [konflux-info] updating namespace I0511 18:10:18.337245 3696 namespace.go:142] [namespace-lister] updating namespace I0511 18:10:18.432964 3696 namespace.go:142] [release-service] updating namespace I0511 18:10:18.533251 3696 namespace.go:142] [segment-bridge] updating namespace W0511 18:10:19.788431 3696 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0511 18:10:20.797045 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0511 18:10:20.798077 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0511 18:10:20.828929 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0511 18:10:20.855223 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0511 18:10:20.855381 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0511 18:10:23.185915 3696 cni.go:452] [konflux-ui/proxy-6557cd967f-pdhwm 4ba5f71be4101efa2c44e16e11e865b424da3961f7e3f0a7a3e2c8ba735a7815 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-6557cd967f-pdhwm 4ba5f71be4101efa2c44e16e11e865b424da3961f7e3f0a7a3e2c8ba735a7815 network default NAD default NAD key ] I0511 18:10:23.195127 3696 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-6557cd967f-pdhwm, hostIfaceName: 4ba5f71be4101ef, network: default, NAD default, SandboxID: "4ba5f71be4101efa2c44e16e11e865b424da3961f7e3f0a7a3e2c8ba735a7815", PCI device ID: , UID: "5df872d8-a182-4922-a662-413cd7e0eb86", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0511 18:10:23.251346 3696 cni.go:473] [konflux-ui/proxy-6557cd967f-pdhwm 4ba5f71be4101efa2c44e16e11e865b424da3961f7e3f0a7a3e2c8ba735a7815 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-6557cd967f-pdhwm 4ba5f71be4101efa2c44e16e11e865b424da3961f7e3f0a7a3e2c8ba735a7815 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:98:a7:b9:07:21\",\"name\":\"4ba5f71be4101ef\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7f155de-bbf9-4d66-ab20-b0d7bde57603\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0511 18:10:23.822742 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0511 18:10:24.835260 3696 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0511 18:10:26.381074 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=78 W0511 18:10:27.859403 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0511 18:10:27.859422 3696 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0511 18:10:27.887267 3696 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0511 18:10:27.887871 3696 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0511 18:10:28.039513 3696 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0511 18:10:28.039529 3696 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0511 18:10:28.482422 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0511 18:10:32.201026 3696 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-p6bpq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:32.201707 3696 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-p6bpq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:10:32.893888 3696 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-p6bpq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:32.893920 3696 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-p6bpq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:10:32.899137 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-p6bpq I0511 18:10:32.899158 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-p6bpq W0511 18:11:04.930517 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0511 18:11:04.930539 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0511 18:11:04.930556 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0511 18:11:04.930561 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0511 18:11:08.122367 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0511 18:11:08.122385 3696 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0511 18:11:13.487280 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0511 18:11:33.476679 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0511 18:11:38.361234 3696 obj_retry.go:498] Detected object konflux-ui/proxy-b94f86758-5r7vh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:11:38.361816 3696 obj_retry.go:498] Detected object konflux-ui/proxy-b94f86758-5r7vh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:11:39.294087 3696 obj_retry.go:492] Detected object konflux-ui/proxy-b94f86758-5r7vh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:11:39.294112 3696 obj_retry.go:492] Detected object konflux-ui/proxy-b94f86758-5r7vh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:11:39.297421 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-b94f86758-5r7vh I0511 18:11:39.297441 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-b94f86758-5r7vh I0511 18:11:43.380278 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=71 I0511 18:11:44.491358 3696 cni.go:452] [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key ] I0511 18:11:44.582604 3696 cni.go:473] [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/test-tsf-konflux 9b8a421142e46f6dcb16d22bd1b9b6b3844e5d1c33940ad543e2e5076e6e7e33 network default NAD default NAD key default], result "{}", err I0511 18:11:44.636091 3696 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:11:44.636127 3696 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0511 18:11:44.637492 3696 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.56 I0511 18:11:44.637531 3696 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:11:44.671549 3696 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:11:44.671571 3696 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:11:44.680299 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0511 18:11:44.680321 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0511 18:12:16.026725 3696 namespace.go:98] [managed-tenant-uvubb] adding namespace I0511 18:12:16.027753 3696 namespace.go:102] [managed-tenant-uvubb] adding namespace took 1.001705ms I0511 18:12:16.071163 3696 namespace.go:142] [managed-tenant-uvubb] updating namespace I0511 18:12:16.097105 3696 namespace.go:142] [managed-tenant-uvubb] updating namespace I0511 18:12:17.108001 3696 namespace.go:142] [managed-tenant-uvubb] updating namespace I0511 18:12:17.230160 3696 namespace.go:142] [managed-tenant-uvubb] updating namespace I0511 18:12:17.329263 3696 namespace.go:142] [managed-tenant-uvubb] updating namespace I0511 18:12:33.505611 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0511 18:12:53.384453 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=637 I0511 18:13:18.506645 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0511 18:14:07.382669 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0511 18:14:49.382513 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=114 I0511 18:15:24.497147 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0511 18:15:38.466100 3696 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0511 18:15:44.485475 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0511 18:16:24.489946 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0511 18:16:43.470668 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0511 18:17:28.469368 3696 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0511 18:17:32.502796 3696 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0511 18:17:46.384177 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=32 I0511 18:18:40.379212 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=55 I0511 18:19:07.386138 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=5 W0511 18:19:24.781769 3696 duration.go:461] Config duration recorder: measurement expired for pod/openshift-console/console-7d6559f477-p77qr I0511 18:19:45.510269 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0511 18:19:48.067725 3696 obj_retry.go:492] Detected object tsf-tpa/create-db-qmtcr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:19:48.067776 3696 obj_retry.go:492] Detected object tsf-tpa/create-db-qmtcr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:19:48.067889 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-qmtcr I0511 18:19:48.067911 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-qmtcr I0511 18:19:52.030189 3696 obj_retry.go:492] Detected object tsf-tpa/migrate-db-klwgs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:19:52.030241 3696 obj_retry.go:492] Detected object tsf-tpa/migrate-db-klwgs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:19:52.035810 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-klwgs I0511 18:19:52.035832 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-klwgs I0511 18:19:56.028455 3696 obj_retry.go:492] Detected object tsf-tpa/create-importers-kc88q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:19:56.028528 3696 obj_retry.go:492] Detected object tsf-tpa/create-importers-kc88q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:19:56.032191 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-kc88q I0511 18:19:56.032216 3696 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-kc88q I0511 18:20:00.193338 3696 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29642060-7mh7c for pod on switch ip-10-0-2-22.ec2.internal I0511 18:20:00.193473 3696 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c I0511 18:20:00.228595 3696 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c] pod update took 35.149684ms I0511 18:20:00.228621 3696 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c] addLogicalPort annotation time took 35.182553ms I0511 18:20:00.229930 3696 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c] addLogicalPort took 36.606558ms, libovsdb time 884.236µs I0511 18:20:00.555488 3696 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key ] I0511 18:20:00.564665 3696 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29642060-7mh7c, hostIfaceName: 4f332d862f8e015, network: default, NAD default, SandboxID: "4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c", PCI device ID: , UID: "a315c644-89f5-443d-b2da-7d998d888805", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0511 18:20:00.623454 3696 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:06:9d:b8:39:46\",\"name\":\"4f332d862f8e015\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/07eb27ba-258e-4365-b8a9-cfb97e860d14\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0511 18:20:04.337036 3696 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key ] I0511 18:20:04.426159 3696 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c 4f332d862f8e015f0dba4c7c855bef0710c5c939a5b2caf291fcd5e18458da5c network default NAD default NAD key default], result "{}", err I0511 18:20:04.478616 3696 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0511 18:20:04.478659 3696 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c I0511 18:20:04.480129 3696 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c, ips: 10.128.0.58 I0511 18:20:04.480157 3696 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0511 18:20:05.361017 3696 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:20:05.361041 3696 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29642060-7mh7c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0511 18:20:09.388124 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=24 I0511 18:20:12.479852 3696 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0511 18:20:18.383980 3696 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=11