+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-108.ec2.internal ++ [[ -n ip-10-0-2-108.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-108.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260318-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260318-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260318-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ cut -f 1 -d . ++ echo 9.6 + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0403 00:03:27.245522305 - disable conntrack on geneve port' I0403 00:03:27.245522305 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0403 00:03:27.265813784 - starting ovnkube-node' I0403 00:03:27.265813784 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-108.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-108.ec2.internal --init-node ip-10-0-2-108.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-108.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0403 00:03:27.303795 3870 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0403 00:03:27.303855 3870 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-f1cb2dbfda.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0403 00:03:27.305425 3870 kube.go:423] Waiting for certificate I0403 00:03:27.305462 3870 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0403 00:03:27.305494 3870 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0403 00:03:27.306282 3870 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0403 00:03:27.336592 3870 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:03:27.336617 3870 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:03:27.339160 3870 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:03:27.341645 3870 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-n7c78" I0403 00:03:27.348585 3870 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-n7c78" I0403 00:03:27.348685 3870 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:03:28.306186 3870 kube.go:430] Certificate found I0403 00:03:28.306758 3870 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0403 00:03:28.307994 3870 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0403 00:03:28.360141 3870 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-03 23:58:27 +0000 UTC" deadline="2026-04-03 19:17:46.216183064 +0000 UTC" I0403 00:03:28.360194 3870 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h14m17.856004122s" I0403 00:03:28.380858 3870 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0403 00:03:28.399304 3870 controller_manager.go:368] Starting the ovnkube controller I0403 00:03:28.399325 3870 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-108.ec2.internal I0403 00:03:28.399391 3870 controller_manager.go:393] NBDB zone sync took: 50.805µs I0403 00:03:28.399403 3870 factory.go:537] Starting watch factory I0403 00:03:28.399505 3870 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.399559 3870 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400176 3870 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400160 3870 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400252 3870 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400266 3870 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400353 3870 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400448 3870 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400458 3870 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.400479 3870 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.401465 3870 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.401557 3870 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.405457 3870 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0403 00:03:28.405581 3870 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0403 00:03:28.405677 3870 factory.go:537] Starting watch factory I0403 00:03:28.420308 3870 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.420715 3870 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.421630 3870 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.424297 3870 ovn_northd.go:121] Registering OVN northd metrics I0403 00:03:28.424938 3870 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.425245 3870 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.425277 3870 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0403 00:03:28.430781 3870 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0403 00:03:28.430803 3870 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0403 00:03:28.430816 3870 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:03:28.448478 3870 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0403 00:03:28.505621 3870 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0403 00:03:28.505658 3870 factory.go:1931] *v1.Pod informer cache synced successfully I0403 00:03:28.505662 3870 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0403 00:03:28.505666 3870 factory.go:1931] *v1.Namespace informer cache synced successfully I0403 00:03:28.505669 3870 factory.go:1931] *v1.Node informer cache synced successfully I0403 00:03:28.505672 3870 factory.go:1931] *v1.Service informer cache synced successfully I0403 00:03:28.505769 3870 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:03:28.505781 3870 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:03:28.505809 3870 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:03:28.505790 3870 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:03:28.509045 3870 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:03:28.509031 3870 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:03:28.510934 3870 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0403 00:03:28.510948 3870 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0403 00:03:28.511005 3870 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.511016 3870 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.511509 3870 factory.go:1931] *v1.Node informer cache synced successfully I0403 00:03:28.511517 3870 factory.go:1931] *v1.Service informer cache synced successfully I0403 00:03:28.511521 3870 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0403 00:03:28.511526 3870 factory.go:1931] *v1.Pod informer cache synced successfully I0403 00:03:28.511530 3870 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0403 00:03:28.511535 3870 factory.go:1931] *v1.Namespace informer cache synced successfully I0403 00:03:28.513417 3870 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.516667 3870 factory.go:1931] *v1.EgressIP informer cache synced successfully I0403 00:03:28.516677 3870 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0403 00:03:28.516684 3870 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0403 00:03:28.516724 3870 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.516737 3870 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.519465 3870 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.522718 3870 factory.go:1931] *v1.EgressIP informer cache synced successfully I0403 00:03:28.522731 3870 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0403 00:03:28.522793 3870 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.522806 3870 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.525238 3870 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.528507 3870 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0403 00:03:28.528520 3870 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0403 00:03:28.528567 3870 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.528580 3870 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.531247 3870 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.534491 3870 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0403 00:03:28.534498 3870 factory.go:1931] *v1.EgressService informer cache synced successfully I0403 00:03:28.534564 3870 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.534577 3870 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.537035 3870 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.540362 3870 factory.go:1931] *v1.EgressService informer cache synced successfully I0403 00:03:28.540362 3870 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0403 00:03:28.540968 3870 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0403 00:03:28.540983 3870 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0403 00:03:28.543502 3870 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0403 00:03:28.545403 3870 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0403 00:03:28.545477 3870 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0403 00:03:28.545563 3870 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.545577 3870 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.545614 3870 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.545629 3870 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.548357 3870 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.548412 3870 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:03:28.550638 3870 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0403 00:03:28.550652 3870 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0403 00:03:28.550659 3870 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0403 00:03:28.550665 3870 factory.go:649] Watch Factory start up complete, took: 151.265612ms I0403 00:03:28.550673 3870 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-108.ec2.internal" zone I0403 00:03:28.555866 3870 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0403 00:03:28.555876 3870 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0403 00:03:28.555883 3870 factory.go:649] Watch Factory start up complete, took: 150.20765ms I0403 00:03:28.555959 3870 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0403 00:03:28.598668 3870 default_node_network_controller.go:709] Initializing the default node network controller I0403 00:03:28.649220 3870 udn_isolation.go:101] Starting UDN host isolation manager I0403 00:03:28.656115 3870 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0403 00:03:28.702166 3870 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0403 00:03:28.702236 3870 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0403 00:03:28.702250 3870 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0403 00:03:28.734451 3870 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0403 00:03:28.734508 3870 default_node_network_controller.go:808] Node ip-10-0-2-108.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0403 00:03:28.740029 3870 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.108"] k8s.ovn.org/zone-name:ip-10-0-2-108.ec2.internal] on node ip-10-0-2-108.ec2.internal I0403 00:03:28.763713 3870 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0403 00:03:28.769609 3870 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0403 00:03:28.769865 3870 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0403 00:03:28.770079 3870 gateway_init.go:263] Preparing Gateway I0403 00:03:28.770088 3870 gateway_shared_intf.go:1658] Creating new gateway I0403 00:03:28.800720 3870 controller_manager.go:427] Waiting for node in zone sync took: 250.036058ms I0403 00:03:28.805042 3870 controller_manager.go:608] Switching to transit router for layer2 networks I0403 00:03:28.805074 3870 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-108.ec2.internal I0403 00:03:28.819731 3870 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0403 00:03:28.825956 3870 controller_manager.go:288] SCTP support detected in OVN I0403 00:03:28.826608 3870 services_controller.go:69] Creating services controller for network=default I0403 00:03:28.826705 3870 udn_enabled_svc.go:80] Waiting for service informer to sync I0403 00:03:28.826950 3870 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0403 00:03:28.826997 3870 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0403 00:03:28.827013 3870 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0403 00:03:28.827027 3870 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0403 00:03:28.827230 3870 udn_enabled_svc.go:95] Performing full resync I0403 00:03:28.827237 3870 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0403 00:03:28.827305 3870 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0403 00:03:28.827326 3870 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 48.236µs I0403 00:03:28.827346 3870 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 28.24µs I0403 00:03:28.827358 3870 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 52.254µs I0403 00:03:28.827366 3870 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0403 00:03:28.827374 3870 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0403 00:03:28.827388 3870 default_network_controller.go:321] Starting the default network controller I0403 00:03:28.827472 3870 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 15.105µs I0403 00:03:28.827542 3870 udn_enabled_svc.go:99] Waiting for handler to sync I0403 00:03:28.827553 3870 udn_enabled_svc.go:104] Starting worker I0403 00:03:28.836430 3870 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0403 00:03:28.836495 3870 repair.go:33] Syncing exgw routes took 45.581µs I0403 00:03:28.836515 3870 default_network_controller.go:407] Starting all the Watchers... I0403 00:03:28.836647 3870 namespace.go:98] [openshift-infra] adding namespace I0403 00:03:28.836681 3870 namespace.go:98] [openshift-monitoring] adding namespace I0403 00:03:28.836695 3870 namespace.go:98] [openshift-marketplace] adding namespace I0403 00:03:28.836699 3870 namespace.go:98] [openshift-backplane-cse] adding namespace I0403 00:03:28.836712 3870 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0403 00:03:28.836729 3870 namespace.go:98] [openshift-apiserver] adding namespace I0403 00:03:28.836734 3870 namespace.go:98] [openshift-network-console] adding namespace I0403 00:03:28.836739 3870 namespace.go:98] [openshift-package-operator] adding namespace I0403 00:03:28.836746 3870 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0403 00:03:28.836747 3870 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0403 00:03:28.836746 3870 namespace.go:98] [openshift-route-controller-manager] adding namespace I0403 00:03:28.836751 3870 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0403 00:03:28.836752 3870 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0403 00:03:28.836762 3870 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0403 00:03:28.836772 3870 namespace.go:98] [openshift-ingress-canary] adding namespace I0403 00:03:28.837260 3870 namespace.go:102] [openshift-infra] adding namespace took 601.141µs I0403 00:03:28.837277 3870 namespace.go:98] [openshift-backplane] adding namespace I0403 00:03:28.837695 3870 namespace.go:102] [openshift-backplane] adding namespace took 411.968µs I0403 00:03:28.837709 3870 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0403 00:03:28.838164 3870 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 449.866µs I0403 00:03:28.838176 3870 namespace.go:98] [openshift-backplane-srep] adding namespace I0403 00:03:28.838641 3870 namespace.go:102] [openshift-monitoring] adding namespace took 1.945319ms I0403 00:03:28.838658 3870 namespace.go:98] [openshift-must-gather-operator] adding namespace I0403 00:03:28.839665 3870 namespace.go:102] [openshift-marketplace] adding namespace took 2.958818ms I0403 00:03:28.839686 3870 namespace.go:98] [openshift-node] adding namespace I0403 00:03:28.840090 3870 namespace.go:102] [openshift-backplane-cse] adding namespace took 3.382638ms I0403 00:03:28.840107 3870 namespace.go:98] [openshift-insights] adding namespace I0403 00:03:28.840632 3870 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 3.906944ms I0403 00:03:28.840650 3870 namespace.go:98] [openshift-network-node-identity] adding namespace I0403 00:03:28.841148 3870 namespace.go:102] [openshift-apiserver] adding namespace took 4.410873ms I0403 00:03:28.841161 3870 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0403 00:03:28.841625 3870 namespace.go:102] [openshift-network-console] adding namespace took 4.883932ms I0403 00:03:28.841643 3870 namespace.go:98] [default] adding namespace I0403 00:03:28.842075 3870 namespace.go:102] [openshift-package-operator] adding namespace took 5.331089ms I0403 00:03:28.842088 3870 namespace.go:98] [openshift-cluster-version] adding namespace I0403 00:03:28.842508 3870 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.755774ms I0403 00:03:28.842523 3870 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0403 00:03:28.842934 3870 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.178896ms I0403 00:03:28.842945 3870 namespace.go:98] [kube-node-lease] adding namespace I0403 00:03:28.843379 3870 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.621548ms I0403 00:03:28.843395 3870 namespace.go:98] [openshift-apiserver-operator] adding namespace I0403 00:03:28.843832 3870 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.072581ms I0403 00:03:28.843845 3870 namespace.go:98] [openshift-dns-operator] adding namespace I0403 00:03:28.844317 3870 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.554663ms I0403 00:03:28.844348 3870 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0403 00:03:28.844837 3870 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.059062ms I0403 00:03:28.844852 3870 namespace.go:98] [openshift-console] adding namespace I0403 00:03:28.845250 3870 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.471039ms I0403 00:03:28.845265 3870 namespace.go:98] [openshift-controller-manager] adding namespace I0403 00:03:28.845756 3870 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.573921ms I0403 00:03:28.845769 3870 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0403 00:03:28.846326 3870 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.66186ms I0403 00:03:28.846340 3870 namespace.go:98] [openshift-operators] adding namespace I0403 00:03:28.846774 3870 namespace.go:102] [openshift-node] adding namespace took 7.083092ms I0403 00:03:28.846786 3870 namespace.go:98] [openshift-logging] adding namespace I0403 00:03:28.847194 3870 namespace.go:102] [openshift-insights] adding namespace took 7.081276ms I0403 00:03:28.847208 3870 namespace.go:98] [openshift-host-network] adding namespace I0403 00:03:28.847594 3870 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.93709ms I0403 00:03:28.847607 3870 namespace.go:98] [openshift-service-ca-operator] adding namespace I0403 00:03:28.847967 3870 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.801484ms I0403 00:03:28.847979 3870 namespace.go:98] [openshift-console-operator] adding namespace I0403 00:03:28.848437 3870 namespace.go:102] [default] adding namespace took 6.786937ms I0403 00:03:28.848451 3870 namespace.go:98] [openshift-dns] adding namespace I0403 00:03:28.848757 3870 namespace.go:102] [openshift-cluster-version] adding namespace took 6.665607ms I0403 00:03:28.848768 3870 namespace.go:98] [kube-public] adding namespace I0403 00:03:28.849174 3870 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.64651ms I0403 00:03:28.849184 3870 namespace.go:98] [openshift-authentication-operator] adding namespace I0403 00:03:28.849686 3870 namespace.go:102] [kube-node-lease] adding namespace took 6.735843ms I0403 00:03:28.849699 3870 namespace.go:98] [openshift-image-registry] adding namespace I0403 00:03:28.850072 3870 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.660175ms I0403 00:03:28.850086 3870 namespace.go:98] [openshift-network-operator] adding namespace I0403 00:03:28.850448 3870 namespace.go:102] [openshift-dns-operator] adding namespace took 6.5984ms I0403 00:03:28.850459 3870 namespace.go:98] [openshift-kube-scheduler] adding namespace I0403 00:03:28.850826 3870 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.474087ms I0403 00:03:28.850837 3870 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0403 00:03:28.851214 3870 namespace.go:102] [openshift-console] adding namespace took 6.357684ms I0403 00:03:28.851225 3870 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0403 00:03:28.851291 3870 iptables.go:108] Creating table: filter chain: FORWARD I0403 00:03:28.851572 3870 namespace.go:102] [openshift-controller-manager] adding namespace took 6.301489ms I0403 00:03:28.851582 3870 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0403 00:03:28.851955 3870 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.181607ms I0403 00:03:28.852377 3870 namespace.go:102] [openshift-operators] adding namespace took 6.032828ms I0403 00:03:28.852388 3870 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0403 00:03:28.852807 3870 namespace.go:102] [openshift-logging] adding namespace took 6.015872ms I0403 00:03:28.852820 3870 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0403 00:03:28.853254 3870 namespace.go:102] [openshift-host-network] adding namespace took 6.041757ms I0403 00:03:28.853265 3870 namespace.go:98] [openshift-config] adding namespace I0403 00:03:28.853679 3870 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.06687ms I0403 00:03:28.853692 3870 namespace.go:98] [openshift-machine-config-operator] adding namespace I0403 00:03:28.854048 3870 namespace.go:102] [openshift-console-operator] adding namespace took 6.064556ms I0403 00:03:28.854071 3870 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0403 00:03:28.854445 3870 namespace.go:102] [openshift-dns] adding namespace took 5.989586ms I0403 00:03:28.854456 3870 namespace.go:98] [openshift-backplane-csm] adding namespace I0403 00:03:28.854820 3870 namespace.go:102] [kube-public] adding namespace took 6.048103ms I0403 00:03:28.854832 3870 namespace.go:98] [kube-system] adding namespace I0403 00:03:28.855288 3870 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.099207ms I0403 00:03:28.855300 3870 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0403 00:03:28.855815 3870 namespace.go:102] [openshift-image-registry] adding namespace took 6.111755ms I0403 00:03:28.855825 3870 namespace.go:98] [openshift-authentication] adding namespace I0403 00:03:28.856214 3870 namespace.go:102] [openshift-network-operator] adding namespace took 6.122691ms I0403 00:03:28.856227 3870 namespace.go:98] [openshift-ingress-operator] adding namespace I0403 00:03:28.856807 3870 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.342324ms I0403 00:03:28.856820 3870 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0403 00:03:28.857264 3870 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.421259ms I0403 00:03:28.857279 3870 namespace.go:98] [openshift-kube-apiserver] adding namespace I0403 00:03:28.857583 3870 iptables.go:108] Creating table: filter chain: OUTPUT I0403 00:03:28.857639 3870 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.408305ms I0403 00:03:28.857657 3870 namespace.go:98] [openshift-machine-api] adding namespace I0403 00:03:28.858083 3870 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.495431ms I0403 00:03:28.858094 3870 namespace.go:98] [openshift-config-operator] adding namespace I0403 00:03:28.858524 3870 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.130721ms I0403 00:03:28.858537 3870 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0403 00:03:28.858906 3870 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.079886ms I0403 00:03:28.858924 3870 namespace.go:98] [openshift-network-diagnostics] adding namespace I0403 00:03:28.859309 3870 namespace.go:102] [openshift-config] adding namespace took 6.039484ms I0403 00:03:28.859321 3870 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0403 00:03:28.859678 3870 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.980179ms I0403 00:03:28.859689 3870 namespace.go:98] [openshift-operators-redhat] adding namespace I0403 00:03:28.860086 3870 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.009047ms I0403 00:03:28.860103 3870 namespace.go:98] [dedicated-admin] adding namespace W0403 00:03:28.860516 3870 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-nxhhs: pod kube-system/global-pull-secret-syncer-nxhhs: no pod IPs found I0403 00:03:28.860563 3870 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.100606ms I0403 00:03:28.860576 3870 namespace.go:98] [openshift-ingress] adding namespace I0403 00:03:28.861046 3870 namespace.go:102] [kube-system] adding namespace took 6.209426ms I0403 00:03:28.861056 3870 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0403 00:03:28.861526 3870 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.220064ms I0403 00:03:28.861541 3870 namespace.go:98] [openshift-customer-monitoring] adding namespace I0403 00:03:28.861947 3870 namespace.go:102] [openshift-authentication] adding namespace took 6.117615ms I0403 00:03:28.862727 3870 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.493165ms I0403 00:03:28.862742 3870 namespace.go:98] [openshift-config-managed] adding namespace I0403 00:03:28.863134 3870 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.304889ms I0403 00:03:28.863149 3870 namespace.go:98] [openshift-multus] adding namespace I0403 00:03:28.863560 3870 gateway_shared_intf.go:1787] Gateway Creation Complete I0403 00:03:28.863568 3870 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.283714ms I0403 00:03:28.863578 3870 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0403 00:03:28.863574 3870 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-108.ec2.internal","mac-address":"0e:63:bb:00:8a:13","ip-addresses":["10.0.2.108/24"],"ip-address":"10.0.2.108/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:b823c8ff-f83c-420f-ba95-628da95256ec k8s.ovn.org/node-encap-ips:["10.0.2.108"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.108/24"} k8s.ovn.org/zone-name:ip-10-0-2-108.ec2.internal] on node ip-10-0-2-108.ec2.internal I0403 00:03:28.863979 3870 namespace.go:102] [openshift-machine-api] adding namespace took 6.313574ms I0403 00:03:28.863993 3870 namespace.go:98] [openshift-backplane-mobb] adding namespace I0403 00:03:28.864447 3870 namespace.go:102] [openshift-config-operator] adding namespace took 6.345645ms I0403 00:03:28.864462 3870 namespace.go:98] [openshift] adding namespace W0403 00:03:28.864811 3870 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-4bpxr: pod openshift-network-diagnostics/network-check-target-4bpxr: no pod IPs found W0403 00:03:28.864822 3870 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-zgkc5: pod openshift-network-diagnostics/network-check-target-zgkc5: no pod IPs found I0403 00:03:28.864855 3870 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.314224ms I0403 00:03:28.865319 3870 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.38955ms I0403 00:03:28.865337 3870 namespace.go:98] [openshift-etcd] adding namespace I0403 00:03:28.865714 3870 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.389519ms I0403 00:03:28.866081 3870 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.3874ms I0403 00:03:28.866092 3870 namespace.go:98] [openshift-backplane-cee] adding namespace I0403 00:03:28.866576 3870 namespace.go:102] [dedicated-admin] adding namespace took 6.465977ms I0403 00:03:28.866592 3870 namespace.go:98] [openshift-backplane-tam] adding namespace I0403 00:03:28.866941 3870 namespace.go:102] [openshift-ingress] adding namespace took 6.360386ms I0403 00:03:28.867353 3870 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.274562ms I0403 00:03:28.867745 3870 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.197588ms W0403 00:03:28.868096 3870 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-9xsl8: pod openshift-multus/network-metrics-daemon-9xsl8: no pod IPs found W0403 00:03:28.868107 3870 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-nwtwd: pod openshift-multus/network-metrics-daemon-nwtwd: no pod IPs found I0403 00:03:28.868144 3870 namespace.go:102] [openshift-config-managed] adding namespace took 5.397593ms I0403 00:03:28.868593 3870 namespace.go:102] [openshift-multus] adding namespace took 5.438251ms I0403 00:03:28.868608 3870 namespace.go:98] [openshift-console-user-settings] adding namespace I0403 00:03:28.868973 3870 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.388851ms I0403 00:03:28.869382 3870 namespace.go:102] [openshift-backplane-mobb] adding namespace took 5.383127ms I0403 00:03:28.869760 3870 namespace.go:102] [openshift] adding namespace took 5.292058ms I0403 00:03:28.869777 3870 namespace.go:98] [open-cluster-management-2peb42nlhreqhm7gegjie4h8sih13gfv] adding namespace I0403 00:03:28.870161 3870 namespace.go:102] [openshift-etcd] adding namespace took 4.819854ms I0403 00:03:28.870555 3870 namespace.go:102] [openshift-backplane-cee] adding namespace took 4.458843ms I0403 00:03:28.870916 3870 namespace.go:102] [openshift-backplane-tam] adding namespace took 4.319413ms I0403 00:03:28.871402 3870 namespace.go:102] [openshift-console-user-settings] adding namespace took 2.788852ms I0403 00:03:28.871731 3870 namespace.go:102] [open-cluster-management-2peb42nlhreqhm7gegjie4h8sih13gfv] adding namespace took 1.948504ms I0403 00:03:28.872110 3870 zone_ic_handler.go:186] Time taken to create transit switch: 277.421µs I0403 00:03:28.872174 3870 master.go:532] Adding or Updating local node "ip-10-0-2-108.ec2.internal" for network "default" I0403 00:03:28.872251 3870 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-147.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-147.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-147.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-147.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-147.ec2.internal I0403 00:03:28.875220 3870 default_network_controller.go:806] Node add failed for ip-10-0-2-108.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-108.ec2.internal, error getting gateway config for node ip-10-0-2-108.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-108.ec2.internal", failed to update chassis to local for local node ip-10-0-2-108.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-108.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-108.ec2.internal] E0403 00:03:28.875273 3870 obj_retry.go:541] Failed to create *v1.Node ip-10-0-2-108.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-108.ec2.internal, error getting gateway config for node ip-10-0-2-108.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-108.ec2.internal", failed to update chassis to local for local node ip-10-0-2-108.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-108.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-108.ec2.internal] I0403 00:03:28.875324 3870 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0403 00:03:28.875348 3870 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0403 00:03:28.875337 3870 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-108.ec2.internal", UID:"71b6b3b9-e05e-441e-bff2-6a88d372c35e", APIVersion:"v1", ResourceVersion:"8457", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-108.ec2.internal, error getting gateway config for node ip-10-0-2-108.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-108.ec2.internal", failed to update chassis to local for local node ip-10-0-2-108.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-108.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-108.ec2.internal] I0403 00:03:28.875369 3870 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0403 00:03:28.875378 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-147.ec2.internal in network "default" I0403 00:03:28.875418 3870 node_tracker.go:235] Node ip-10-0-0-147.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-147.ec2.internal" I0403 00:03:28.875437 3870 node_tracker.go:169] Node ip-10-0-0-147.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:28.875449 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:28.875464 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-108.ec2.internal in network "default" I0403 00:03:28.875480 3870 node_tracker.go:235] Node ip-10-0-2-108.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-108.ec2.internal" I0403 00:03:28.875492 3870 node_tracker.go:169] Node ip-10-0-2-108.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:28.875499 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:28.879490 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-108.ec2.internal in network "default" I0403 00:03:28.879503 3870 master.go:532] Adding or Updating local node "ip-10-0-2-108.ec2.internal" for network "default" I0403 00:03:28.879550 3870 node_tracker.go:169] Node ip-10-0-2-108.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:28.879566 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:28.883321 3870 default_node_network_controller.go:518] Node connection status = connected W0403 00:03:28.884086 3870 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-9xsl8: pod openshift-multus/network-metrics-daemon-9xsl8: no pod IPs found W0403 00:03:28.884102 3870 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-nxhhs: pod kube-system/global-pull-secret-syncer-nxhhs: no pod IPs found W0403 00:03:28.884111 3870 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-zgkc5: pod openshift-network-diagnostics/network-check-target-zgkc5: no pod IPs found I0403 00:03:28.885188 3870 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-108.ec2.internal for the network default I0403 00:03:28.976034 3870 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0403 00:03:28.976057 3870 services_controller.go:208] Setting up event handlers for services for network=default I0403 00:03:28.976175 3870 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0403 00:03:28.976229 3870 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0403 00:03:28.976246 3870 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0403 00:03:29.076459 3870 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0403 00:03:29.076485 3870 repair.go:60] Starting repairing loop for services I0403 00:03:29.076591 3870 repair.go:131] Deleted 0 stale service LBs I0403 00:03:29.076618 3870 repair.go:137] Deleted 0 stale Chassis Template Vars I0403 00:03:29.076644 3870 repair.go:62] Finished repairing loop for services: 160.824µs I0403 00:03:29.076681 3870 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0403 00:03:29.076696 3870 services_controller.go:254] Starting workers for network=default I0403 00:03:29.077012 3870 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-zgkc5] creating logical port openshift-network-diagnostics_network-check-target-zgkc5 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:29.077106 3870 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-9xsl8] creating logical port openshift-multus_network-metrics-daemon-9xsl8 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:29.077142 3870 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-nwtwd, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nwtwd: suppressed error logged: pod openshift-multus/network-metrics-daemon-nwtwd: no pod IPs found I0403 00:03:29.077154 3870 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-4bpxr, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-4bpxr: suppressed error logged: pod openshift-network-diagnostics/network-check-target-4bpxr: no pod IPs found I0403 00:03:29.077185 3870 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-nxhhs] creating logical port kube-system_global-pull-secret-syncer-nxhhs for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:29.077244 3870 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-zgkc5 I0403 00:03:29.077266 3870 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-9xsl8 I0403 00:03:29.077275 3870 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-nxhhs I0403 00:03:29.093627 3870 pod.go:62] [kube-system/global-pull-secret-syncer-nxhhs] pod update took 16.366375ms I0403 00:03:29.093650 3870 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-nxhhs] addLogicalPort annotation time took 16.401332ms I0403 00:03:29.094559 3870 pods.go:271] [kube-system/global-pull-secret-syncer-nxhhs] addLogicalPort took 17.382135ms, libovsdb time 672.07µs I0403 00:03:29.095950 3870 pod.go:62] [openshift-network-diagnostics/network-check-target-zgkc5] pod update took 18.819259ms I0403 00:03:29.095968 3870 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-zgkc5] addLogicalPort annotation time took 18.844523ms I0403 00:03:29.096827 3870 pods.go:271] [openshift-network-diagnostics/network-check-target-zgkc5] addLogicalPort took 19.829231ms, libovsdb time 620.663µs I0403 00:03:29.096909 3870 pod.go:62] [openshift-multus/network-metrics-daemon-9xsl8] pod update took 19.725671ms I0403 00:03:29.096925 3870 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-9xsl8] addLogicalPort annotation time took 19.745621ms I0403 00:03:29.098273 3870 pods.go:271] [openshift-multus/network-metrics-daemon-9xsl8] addLogicalPort took 21.183107ms, libovsdb time 1.066807ms I0403 00:03:29.098631 3870 admin_network_policy_controller.go:227] Starting controller default-network-controller I0403 00:03:29.098649 3870 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0403 00:03:29.098661 3870 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0403 00:03:29.098665 3870 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0403 00:03:29.098836 3870 repair.go:29] Repairing admin network policies took 166.194µs I0403 00:03:29.098930 3870 repair.go:92] Repairing baseline admin network policies took 86.432µs I0403 00:03:29.099318 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0403 00:03:29.099338 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0403 00:03:29.100955 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0403 00:03:29.100977 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0403 00:03:29.100987 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0403 00:03:29.102732 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0403 00:03:29.102753 3870 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0403 00:03:29.102764 3870 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0403 00:03:29.104668 3870 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0403 00:03:29.104689 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0403 00:03:29.104700 3870 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0403 00:03:29.106506 3870 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0403 00:03:29.106525 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0403 00:03:29.106533 3870 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0403 00:03:29.107050 3870 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0403 00:03:29.107080 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0403 00:03:29.107090 3870 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0403 00:03:29.107545 3870 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0403 00:03:29.107562 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0403 00:03:29.107940 3870 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0403 00:03:29.107953 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0403 00:03:29.109031 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0403 00:03:29.109049 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0403 00:03:29.109057 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0403 00:03:29.110310 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0403 00:03:29.110327 3870 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0403 00:03:29.110336 3870 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0403 00:03:29.111835 3870 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0403 00:03:29.111855 3870 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0403 00:03:29.111864 3870 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0403 00:03:29.112710 3870 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0403 00:03:29.112728 3870 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0403 00:03:29.112739 3870 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0403 00:03:29.114857 3870 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0403 00:03:29.114875 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0403 00:03:29.114882 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0403 00:03:29.115378 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0403 00:03:29.115395 3870 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0403 00:03:29.115403 3870 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0403 00:03:29.115853 3870 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0403 00:03:29.115868 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0403 00:03:29.115876 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0403 00:03:29.116414 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0403 00:03:29.116428 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0403 00:03:29.116436 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0403 00:03:29.116697 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0403 00:03:29.116715 3870 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0403 00:03:29.116725 3870 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0403 00:03:29.117717 3870 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0403 00:03:29.117734 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0403 00:03:29.117742 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0403 00:03:29.118219 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0403 00:03:29.118235 3870 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0403 00:03:29.118240 3870 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0403 00:03:29.118473 3870 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0403 00:03:29.118486 3870 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0403 00:03:29.118493 3870 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0403 00:03:29.120021 3870 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0403 00:03:29.120037 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0403 00:03:29.120045 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0403 00:03:29.120539 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0403 00:03:29.120560 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0403 00:03:29.120568 3870 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0403 00:03:29.121021 3870 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0403 00:03:29.121039 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0403 00:03:29.121046 3870 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0403 00:03:29.121325 3870 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0403 00:03:29.121340 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0403 00:03:29.121366 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0403 00:03:29.122052 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0403 00:03:29.122084 3870 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0403 00:03:29.122099 3870 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0403 00:03:29.123314 3870 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0403 00:03:29.123332 3870 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0403 00:03:29.123338 3870 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0403 00:03:29.123603 3870 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0403 00:03:29.123619 3870 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0403 00:03:29.123628 3870 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0403 00:03:29.124593 3870 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0403 00:03:29.124608 3870 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0403 00:03:29.124617 3870 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0403 00:03:29.125792 3870 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0403 00:03:29.125813 3870 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0403 00:03:29.125822 3870 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0403 00:03:29.126797 3870 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0403 00:03:29.126816 3870 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0403 00:03:29.126831 3870 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0403 00:03:29.127321 3870 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0403 00:03:29.131441 3870 egressfirewall.go:319] Starting EgressFirewall controller I0403 00:03:29.131474 3870 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0403 00:03:29.131497 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0403 00:03:29.131502 3870 controller.go:133] Adding controller egress-firewall-controller event handlers I0403 00:03:29.131508 3870 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0403 00:03:29.131528 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0403 00:03:29.131539 3870 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0403 00:03:29.131828 3870 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0403 00:03:29.131842 3870 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0403 00:03:29.131850 3870 egressqos.go:193] Setting up event handlers for EgressQoS I0403 00:03:29.131933 3870 egressfirewall.go:628] Syncing node "ip-10-0-0-147.ec2.internal" for egress firewall I0403 00:03:29.131945 3870 egressfirewall.go:628] Syncing node "ip-10-0-2-108.ec2.internal" for egress firewall I0403 00:03:29.131952 3870 egressqos.go:245] Starting EgressQoS Controller I0403 00:03:29.131960 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0403 00:03:29.131966 3870 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0403 00:03:29.131972 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0403 00:03:29.131976 3870 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0403 00:03:29.131981 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0403 00:03:29.131985 3870 shared_informer.go:356] "Caches are synced" controller="egressqos" I0403 00:03:29.131988 3870 egressqos.go:259] Repairing EgressQoSes I0403 00:03:29.131992 3870 egressqos.go:399] Starting repairing loop for egressqos I0403 00:03:29.132054 3870 egressqos.go:401] Finished repairing loop for egressqos: 61.559µs I0403 00:03:29.132086 3870 egressservice_zone.go:132] Setting up event handlers for Egress Services I0403 00:03:29.132112 3870 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-108.ec2.internal I0403 00:03:29.132167 3870 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-108.ec2.internal : 55.295µs I0403 00:03:29.132262 3870 egressservice_zone.go:210] Starting Egress Services Controller I0403 00:03:29.132276 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0403 00:03:29.132289 3870 shared_informer.go:356] "Caches are synced" controller="egressservices" I0403 00:03:29.132295 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0403 00:03:29.132300 3870 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0403 00:03:29.132305 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0403 00:03:29.132310 3870 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0403 00:03:29.132334 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0403 00:03:29.132340 3870 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0403 00:03:29.132346 3870 egressservice_zone.go:228] Repairing Egress Services I0403 00:03:29.132614 3870 master_controller.go:88] Starting Admin Policy Based Route Controller I0403 00:03:29.132627 3870 external_controller.go:278] Starting Admin Policy Based Route Controller I0403 00:03:29.132638 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-147.ec2.internal I0403 00:03:29.132647 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-147.ec2.internal: 10.821µs I0403 00:03:29.132653 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-108.ec2.internal I0403 00:03:29.132657 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-108.ec2.internal: 3.872µs I0403 00:03:29.132747 3870 default_network_controller.go:568] Completing all the Watchers took 296.223892ms I0403 00:03:29.132763 3870 default_network_controller.go:572] Starting unidling controllers I0403 00:03:29.132774 3870 unidle.go:46] Registering OVN SB ControllerEvent handler I0403 00:03:29.132785 3870 unidle.go:63] Populating Initial ContollerEvent events I0403 00:03:29.132802 3870 unidle.go:79] Setting up event handlers for services I0403 00:03:29.132861 3870 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0403 00:03:29.383646 3870 default_node_network_controller.go:518] Node connection status = connected I0403 00:03:29.884366 3870 default_node_network_controller.go:518] Node connection status = connected I0403 00:03:30.383575 3870 default_node_network_controller.go:518] Node connection status = connected I0403 00:03:30.708869 3870 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0403 00:03:30.884611 3870 default_node_network_controller.go:518] Node connection status = connected I0403 00:03:30.901921 3870 bridgeconfig.go:537] Gateway is ready I0403 00:03:30.901941 3870 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0403 00:03:30.913502 3870 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.108. Updating OVN encap IP. I0403 00:03:30.918791 3870 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.108 - it is already configured I0403 00:03:30.918802 3870 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.108/24:{} 172.20.0.1/32:{}]. Updating annotations. I0403 00:03:30.918991 3870 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.108/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-108.ec2.internal","mac-address":"0e:63:bb:00:8a:13","ip-addresses":["10.0.2.108/24"],"ip-address":"10.0.2.108/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:b823c8ff-f83c-420f-ba95-628da95256ec k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.108/24"}] on node ip-10-0-2-108.ec2.internal I0403 00:03:30.935678 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-108.ec2.internal I0403 00:03:30.935696 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-108.ec2.internal: 25.273µs I0403 00:03:30.935702 3870 master.go:532] Adding or Updating local node "ip-10-0-2-108.ec2.internal" for network "default" I0403 00:03:30.935705 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-108.ec2.internal in network "default" I0403 00:03:30.935745 3870 egressfirewall.go:628] Syncing node "ip-10-0-2-108.ec2.internal" for egress firewall I0403 00:03:30.935757 3870 node_tracker.go:169] Node ip-10-0-2-108.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:30.935769 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:30.936174 3870 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.108 I0403 00:03:30.936302 3870 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-108.ec2.internal I0403 00:03:30.940407 3870 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-108.ec2.internal. Updating no re-route policies I0403 00:03:30.940468 3870 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-29586240-nt956 I0403 00:03:30.940480 3870 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-nznpj I0403 00:03:30.940485 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-4bpxr I0403 00:03:30.940474 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-zgkc5 I0403 00:03:30.940495 3870 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-nznpj I0403 00:03:30.940497 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-4bpxr I0403 00:03:30.940497 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-zgkc5 I0403 00:03:30.940515 3870 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-nznpj after 0 failed attempt(s) I0403 00:03:30.940528 3870 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-108.ec2.internal I0403 00:03:30.940534 3870 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-108.ec2.internal I0403 00:03:30.940535 3870 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-nxhhs I0403 00:03:30.940535 3870 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-zgkc5] creating logical port openshift-network-diagnostics_network-check-target-zgkc5 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:30.940541 3870 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-nxhhs I0403 00:03:30.940541 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-dgb9f I0403 00:03:30.940545 3870 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-108.ec2.internal after 0 failed attempt(s) I0403 00:03:30.940528 3870 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-4bpxr, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-4bpxr: suppressed error logged: pod openshift-network-diagnostics/network-check-target-4bpxr: no pod IPs found I0403 00:03:30.940555 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-dgb9f I0403 00:03:30.940566 3870 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-pc9gg I0403 00:03:30.940564 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-108.ec2.internal I0403 00:03:30.940576 3870 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-pc9gg I0403 00:03:30.940577 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-dgb9f after 0 failed attempt(s) I0403 00:03:30.940579 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-9xsl8 I0403 00:03:30.940581 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-108.ec2.internal I0403 00:03:30.940577 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-9mlfl I0403 00:03:30.940585 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-9xsl8 I0403 00:03:30.940587 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nwtwd I0403 00:03:30.940591 3870 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-pc9gg after 0 failed attempt(s) I0403 00:03:30.940595 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-9mlfl I0403 00:03:30.940591 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gd6dz I0403 00:03:30.940597 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-108.ec2.internal after 0 failed attempt(s) I0403 00:03:30.940600 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-6jmt6 I0403 00:03:30.940603 3870 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-9xsl8] creating logical port openshift-multus_network-metrics-daemon-9xsl8 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:30.940604 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gd6dz I0403 00:03:30.940568 3870 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-nxhhs] creating logical port kube-system_global-pull-secret-syncer-nxhhs for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:30.940608 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-6jmt6 I0403 00:03:30.940614 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-9mlfl after 0 failed attempt(s) I0403 00:03:30.940614 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2h9qw I0403 00:03:30.940616 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-phzp9 I0403 00:03:30.940620 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-gd6dz after 0 failed attempt(s) I0403 00:03:30.940627 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-6jmt6 after 0 failed attempt(s) I0403 00:03:30.940629 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-phzp9 I0403 00:03:30.940597 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nwtwd I0403 00:03:30.940631 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-x2rnh I0403 00:03:30.940652 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-x2rnh I0403 00:03:30.940661 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-phzp9 after 0 failed attempt(s) I0403 00:03:30.940489 3870 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-29586240-nt956 I0403 00:03:30.940679 3870 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-29586240-nt956 after 0 failed attempt(s) I0403 00:03:30.940680 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-x2rnh after 0 failed attempt(s) I0403 00:03:30.940622 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2h9qw I0403 00:03:30.940702 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2h9qw after 0 failed attempt(s) I0403 00:03:30.940721 3870 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-nwtwd, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nwtwd: suppressed error logged: pod openshift-multus/network-metrics-daemon-nwtwd: no pod IPs found I0403 00:03:30.940603 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-6ct6g I0403 00:03:30.940735 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-6ct6g I0403 00:03:30.940751 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-6ct6g after 0 failed attempt(s) I0403 00:03:30.941217 3870 pods.go:271] [openshift-multus/network-metrics-daemon-9xsl8] addLogicalPort took 613.482µs, libovsdb time 303.228µs I0403 00:03:30.941231 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-9xsl8 after 0 failed attempt(s) I0403 00:03:30.941299 3870 pods.go:271] [kube-system/global-pull-secret-syncer-nxhhs] addLogicalPort took 740.388µs, libovsdb time 370.016µs I0403 00:03:30.941310 3870 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-nxhhs after 0 failed attempt(s) I0403 00:03:30.941377 3870 pods.go:271] [openshift-network-diagnostics/network-check-target-zgkc5] addLogicalPort took 853.016µs, libovsdb time 442.507µs I0403 00:03:30.941389 3870 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-zgkc5 after 0 failed attempt(s) I0403 00:03:30.953359 3870 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0403 00:03:30.965355 3870 iptables.go:108] Creating table: mangle chain: OUTPUT I0403 00:03:30.968616 3870 iptables.go:108] Creating table: nat chain: OUTPUT I0403 00:03:30.989649 3870 iptables.go:108] Creating table: nat chain: PREROUTING I0403 00:03:31.039469 3870 iptables.go:108] Creating table: filter chain: FORWARD I0403 00:03:31.055979 3870 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0403 00:03:31.056143 3870 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0403 00:03:31.056182 3870 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0403 00:03:31.056246 3870 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0403 00:03:31.056301 3870 gateway_init.go:325] Gateway and management port readiness took 2.17696155s I0403 00:03:31.056350 3870 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0403 00:03:31.056389 3870 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0403 00:03:31.056406 3870 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0403 00:03:31.056417 3870 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0403 00:03:31.056478 3870 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0403 00:03:31.056554 3870 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.139µs I0403 00:03:31.056763 3870 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0403 00:03:31.056794 3870 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 22.827µs I0403 00:03:31.056802 3870 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 40.866µs I0403 00:03:31.056808 3870 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0403 00:03:31.056819 3870 nad_controller.go:173] [node-nad-controller NAD controller]: started I0403 00:03:31.056826 3870 default_node_network_controller.go:898] Starting the default node network controller I0403 00:03:31.056832 3870 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0403 00:03:31.056923 3870 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 17.174µs I0403 00:03:31.057013 3870 default_node_network_controller.go:1449] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0403 00:03:31.057022 3870 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0403 00:03:31.057048 3870 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0007e4860 0xc0002ed9d0 0xc0002eda40 0xc0002edab0 0xc0002edb20 0xc0002edb90 0xc0002edc00 0xc000c3c2d0 0xc0002edce0 0xc0002edd50 0xc0005de410 0xc00652f2d0 0xc0002ede30 map[0x2f8f0c0:0xc00221ad20 0x2f8f600:0xc00221aea0 0x2f90080:0xc00241b620 0x2f90320:0xc00221acc0 0x2f90b00:0xc00221ad80 0x2f90da0:0xc00221ade0 0x2f91040:0xc00221ae40 0x2f91820:0xc00221b380 0x2f91ac0:0xc00221b3e0 0x2fbc3e0:0xc001705080 0x2fbc780:0xc001705320 0x2fbd260:0xc000c3e060 0x2fbebc0:0xc001bca000 0x2fbf6a0:0xc001705380 0x2fc0520:0xc001bca060] 0xc0000d2700 0} I0403 00:03:31.057120 3870 gateway.go:151] Starting gateway service sync I0403 00:03:31.057573 3870 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0403 00:03:31.059664 3870 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0403 00:03:31.068737 3870 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0403 00:03:31.069842 3870 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0403 00:03:31.070927 3870 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0403 00:03:31.105225 3870 gateway.go:168] Gateway service sync done. Time taken: 48.093651ms I0403 00:03:31.105528 3870 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30848 I0403 00:03:31.105996 3870 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0007e4860 0xc0002ed9d0 0xc0002eda40 0xc0002edab0 0xc0002edb20 0xc0002edb90 0xc0002edc00 0xc000c3c2d0 0xc0002edce0 0xc0002edd50 0xc0005de410 0xc00652f2d0 0xc0002ede30 map[0x2f8f0c0:0xc00221ad20 0x2f8f600:0xc00221aea0 0x2f90080:0xc00241b620 0x2f90320:0xc00221acc0 0x2f90b00:0xc00221ad80 0x2f90da0:0xc00221ade0 0x2f91040:0xc00221ae40 0x2f91820:0xc00221b380 0x2f91ac0:0xc00221b3e0 0x2fbc3e0:0xc001705080 0x2fbc780:0xc001705320 0x2fbd260:0xc000c3e060 0x2fbebc0:0xc001bca000 0x2fbf6a0:0xc001705380 0x2fc0520:0xc001bca060] 0xc0000d2700 0} I0403 00:03:31.106451 3870 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0007e4860 0xc0002ed9d0 0xc0002eda40 0xc0002edab0 0xc0002edb20 0xc0002edb90 0xc0002edc00 0xc000c3c2d0 0xc0002edce0 0xc0002edd50 0xc0005de410 0xc00652f2d0 0xc0002ede30 map[0x2f8f0c0:0xc00221ad20 0x2f8f600:0xc00221aea0 0x2f90080:0xc00241b620 0x2f90320:0xc00221acc0 0x2f90b00:0xc00221ad80 0x2f90da0:0xc00221ade0 0x2f91040:0xc00221ae40 0x2f91820:0xc00221b380 0x2f91ac0:0xc00221b3e0 0x2fbc3e0:0xc001705080 0x2fbc780:0xc001705320 0x2fbd260:0xc000c3e060 0x2fbebc0:0xc001bca000 0x2fbf6a0:0xc001705380 0x2fc0520:0xc001bca060] 0xc0000d2700 0} I0403 00:03:31.106487 3870 gateway.go:519] Reconciling gateway with updates I0403 00:03:31.111870 3870 gateway.go:326] Spawning Conntrack Rule Check Thread I0403 00:03:31.111955 3870 default_node_network_controller.go:943] Gateway and management port readiness took 54.926762ms I0403 00:03:31.111958 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:03:31.111966 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:03:31.111973 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:03:31.111987 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:03:31.111988 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:03:31.111991 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:03:31.112000 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0403 00:03:31.111996 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0403 00:03:31.112009 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:03:31.112009 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:03:31.112016 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:03:31.112016 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0403 00:03:31.112023 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:03:31.112018 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:03:31.112027 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:03:31.112025 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:03:31.112034 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:03:31.112039 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:03:31.112041 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:03:31.112042 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0403 00:03:31.112049 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0403 00:03:31.112047 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:03:31.112081 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:03:31.112118 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0403 00:03:31.112119 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0403 00:03:31.112141 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0403 00:03:31.112151 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:03:31.112157 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0403 00:03:31.112161 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:03:31.112169 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0403 00:03:31.112190 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:03:31.111951 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:03:31.112203 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0403 00:03:31.112203 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:03:31.112152 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0403 00:03:31.112210 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:03:31.112218 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:03:31.112152 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0403 00:03:31.112228 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:03:31.112243 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:03:31.112248 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0403 00:03:31.112254 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:03:31.112268 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0403 00:03:31.112221 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0403 00:03:31.112307 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0403 00:03:31.112312 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:03:31.112339 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0403 00:03:31.112347 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:03:31.112352 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0403 00:03:31.112352 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0403 00:03:31.112355 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:03:31.112368 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:03:31.112375 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:03:31.112382 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:03:31.112381 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:03:31.112389 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:03:31.112391 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:03:31.112409 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0403 00:03:31.112414 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:03:31.112436 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0403 00:03:31.112421 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:03:31.112448 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0403 00:03:31.112454 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0403 00:03:31.112468 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0403 00:03:31.112473 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:03:31.112479 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0403 00:03:31.112487 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:03:31.112486 3870 port_claim.go:71] Svc openshift-ingress/router-default: port 32571 is already open, no action needed I0403 00:03:31.112486 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0403 00:03:31.112492 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0403 00:03:31.112497 3870 port_claim.go:71] Svc openshift-ingress/router-default: port 30980 is already open, no action needed I0403 00:03:31.112500 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0403 00:03:31.112506 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0403 00:03:31.112528 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:03:31.112367 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:03:31.112544 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:03:31.112548 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0403 00:03:31.112554 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0403 00:03:31.112474 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:03:31.112566 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:03:31.112448 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:03:31.112572 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:03:31.112586 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:03:31.112550 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0403 00:03:31.112456 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:03:31.112632 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:03:31.112423 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:03:31.112654 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:03:31.112660 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:03:31.112678 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:03:31.112685 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:03:31.112542 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0403 00:03:31.112699 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:03:31.112685 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:03:31.112705 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:03:31.112717 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:03:31.112739 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:03:31.112754 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:03:31.112760 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0403 00:03:31.112357 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:03:31.112775 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:03:31.112773 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:03:31.112773 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:03:31.112782 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:03:31.112787 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:03:31.112787 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0403 00:03:31.112788 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:03:31.112797 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0403 00:03:31.112776 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:03:31.112830 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:03:31.112776 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:03:31.112787 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0403 00:03:31.112859 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0403 00:03:31.112774 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:03:31.112874 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:03:31.113631 3870 node_ip_handler_linux.go:160] Node IP manager is running W0403 00:03:31.158169 3870 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0403 00:03:31.158391 3870 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0403 00:03:31.158671 3870 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0403 00:03:31.168490 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0403 00:03:31.168500 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0403 00:03:31.168515 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0403 00:03:31.168525 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0403 00:03:31.168532 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0403 00:03:31.168538 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0403 00:03:31.168544 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0403 00:03:31.168550 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0403 00:03:31.168558 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0403 00:03:31.168564 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0403 00:03:31.168573 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0403 00:03:31.168578 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0403 00:03:31.168584 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0403 00:03:31.209184 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0403 00:03:31.209215 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0403 00:03:31.209224 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0403 00:03:31.209232 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0403 00:03:31.209239 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0403 00:03:31.209245 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0403 00:03:32.817967 3870 default_node_network_controller.go:1369] Starting node controller node sync W0403 00:03:32.817983 3870 egressip_healthcheck.go:76] Health checking using insecure connection I0403 00:03:32.818051 3870 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0403 00:03:32.829700 3870 default_node_network_controller.go:1411] Node controller node sync done. Time taken: 11.703116ms I0403 00:03:32.829918 3870 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0403 00:03:32.830245 3870 egressservice_node.go:93] Setting up event handlers for Egress Services I0403 00:03:32.830435 3870 egressservice_node.go:183] Starting Egress Services Controller I0403 00:03:32.830460 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0403 00:03:32.830477 3870 shared_informer.go:356] "Caches are synced" controller="egressservices" I0403 00:03:32.830488 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0403 00:03:32.830497 3870 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0403 00:03:32.830507 3870 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0403 00:03:32.830516 3870 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0403 00:03:32.830523 3870 egressservice_node.go:197] Repairing Egress Services I0403 00:03:32.841121 3870 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0403 00:03:32.841133 3870 external_controller.go:278] Starting Admin Policy Based Route Controller I0403 00:03:32.841325 3870 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0403 00:03:32.841337 3870 link_network_manager.go:117] Link manager is running I0403 00:03:32.841351 3870 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0403 00:03:32.841448 3870 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0403 00:03:32.841896 3870 vrf_manager.go:128] VRF manager is running I0403 00:03:32.842015 3870 node_controller_manager.go:437] Removing flows to drop GARP I0403 00:03:32.842033 3870 gateway.go:519] Reconciling gateway with updates I0403 00:03:32.869227 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:03:32.869248 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:03:32.869247 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:03:32.869262 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:03:32.869266 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0403 00:03:32.869263 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:03:32.869274 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0403 00:03:32.869274 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:03:32.869284 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:03:32.869287 3870 port_claim.go:71] Svc openshift-ingress/router-default: port 32571 is already open, no action needed I0403 00:03:32.869289 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:03:32.869297 3870 port_claim.go:71] Svc openshift-ingress/router-default: port 30980 is already open, no action needed I0403 00:03:32.869300 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:03:32.869315 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:03:32.869323 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:03:32.869254 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:03:32.869333 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:03:32.869342 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:03:32.869348 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:03:32.869352 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:03:32.869364 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0403 00:03:32.869371 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0403 00:03:32.869374 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0403 00:03:32.869233 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0403 00:03:32.869382 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0403 00:03:32.869270 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:03:32.869383 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:03:32.869390 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0403 00:03:32.869390 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:03:32.869398 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0403 00:03:32.869404 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:03:32.869368 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:03:32.869408 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:03:32.869413 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0403 00:03:32.869418 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:03:32.869421 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:03:32.869428 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:03:32.869430 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:03:32.869437 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:03:32.869440 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:03:32.869394 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:03:32.869445 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:03:32.869451 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:03:32.869458 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:03:32.869462 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0403 00:03:32.869467 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0403 00:03:32.869468 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:03:32.869352 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0403 00:03:32.869479 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:03:32.869480 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:03:32.869482 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:03:32.869488 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:03:32.869491 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:03:32.869502 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:03:32.869520 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:03:32.869529 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:03:32.869537 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:03:32.869540 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:03:32.869540 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:03:32.869548 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:03:32.869504 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:03:32.869553 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:03:32.869558 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:03:32.869555 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0403 00:03:32.869560 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0403 00:03:32.869563 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:03:32.869572 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0403 00:03:32.869573 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:03:32.869572 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:03:32.869582 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:03:32.869584 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:03:32.869563 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0403 00:03:32.869599 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:03:32.869549 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:03:32.869609 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:03:32.869609 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:03:32.869611 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:03:32.869602 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0403 00:03:32.869432 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:03:32.869620 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:03:32.869626 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0403 00:03:32.869635 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:03:32.869647 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:03:32.869619 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:03:32.869655 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0403 00:03:32.869656 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:03:32.869667 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:03:32.869676 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:03:32.869608 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0403 00:03:32.869689 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0403 00:03:32.869638 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:03:32.869722 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:03:32.869845 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:03:32.869861 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:03:32.869869 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0403 00:03:32.869885 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:03:32.869898 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:03:32.869945 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:03:32.869955 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:03:32.870129 3870 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0403 00:03:32.870141 3870 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0403 00:03:32.912654 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0403 00:03:32.912664 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0403 00:03:32.912683 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0403 00:03:32.912689 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0403 00:03:32.912697 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0403 00:03:32.912705 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0403 00:03:32.912712 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0403 00:03:32.912728 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0403 00:03:32.953551 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0403 00:03:32.953585 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0403 00:03:32.953592 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0403 00:03:32.953599 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0403 00:03:32.953605 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0403 00:03:32.953613 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0403 00:03:32.953618 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0403 00:03:32.953624 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0403 00:03:32.953629 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0403 00:03:32.953635 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0403 00:03:32.953641 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0403 00:03:32.953649 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0403 00:03:32.953654 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0403 00:03:32.953660 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0403 00:03:32.953666 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0403 00:03:32.953672 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0403 00:03:32.953677 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0403 00:03:32.953691 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0403 00:03:32.953712 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0403 00:03:32.953718 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0403 00:03:32.953728 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0403 00:03:32.953739 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0403 00:03:32.953746 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0403 00:03:32.953752 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0403 00:03:32.992771 3870 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0403 00:03:34.547416 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-108.ec2.internal I0403 00:03:34.547449 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-108.ec2.internal: 39.381µs I0403 00:03:34.577279 3870 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c687859fb-z2cst] creating logical port openshift-image-registry_image-registry-7c687859fb-z2cst for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.577336 3870 base_network_controller_pods.go:487] [default/openshift-image-registry/image-pruner-29586240-mslpc] creating logical port openshift-image-registry_image-pruner-29586240-mslpc for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.577482 3870 kube.go:256] Updating pod openshift-image-registry/image-pruner-29586240-mslpc I0403 00:03:34.577495 3870 kube.go:256] Updating pod openshift-image-registry/image-registry-7c687859fb-z2cst I0403 00:03:34.590009 3870 pod.go:62] [openshift-image-registry/image-pruner-29586240-mslpc] pod update took 12.588703ms I0403 00:03:34.590034 3870 base_network_controller_pods.go:937] [default/openshift-image-registry/image-pruner-29586240-mslpc] addLogicalPort annotation time took 12.617376ms I0403 00:03:34.591370 3870 pods.go:271] [openshift-image-registry/image-pruner-29586240-mslpc] addLogicalPort took 14.048719ms, libovsdb time 975.8µs I0403 00:03:34.598871 3870 pod.go:62] [openshift-image-registry/image-registry-7c687859fb-z2cst] pod update took 21.452288ms I0403 00:03:34.598891 3870 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7c687859fb-z2cst] addLogicalPort annotation time took 21.477678ms I0403 00:03:34.599994 3870 pods.go:271] [openshift-image-registry/image-registry-7c687859fb-z2cst] addLogicalPort took 22.735098ms, libovsdb time 788.782µs I0403 00:03:34.610913 3870 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-75b48889dc-z74x2 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.611027 3870 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2 I0403 00:03:34.615752 3870 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm] creating logical port openshift-service-ca-operator_service-ca-operator-54f5cf58db-5xbxm for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.615798 3870 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-98c9d9db4-dgm4q for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.615930 3870 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q I0403 00:03:34.615933 3870 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm I0403 00:03:34.622498 3870 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-6465cb84c-bmj99] creating logical port openshift-network-console_networking-console-plugin-6465cb84c-bmj99 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.622612 3870 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-6465cb84c-bmj99 I0403 00:03:34.622759 3870 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6f4dc9bddb-tv5j7] creating logical port openshift-insights_insights-operator-6f4dc9bddb-tv5j7 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.622906 3870 kube.go:256] Updating pod openshift-insights/insights-operator-6f4dc9bddb-tv5j7 I0403 00:03:34.623042 3870 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-58965f67cd-9ck8k] creating logical port openshift-ingress_router-default-58965f67cd-9ck8k for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.623106 3870 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-57d9b95648-7swm2] creating logical port openshift-console-operator_console-operator-57d9b95648-7swm2 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.623176 3870 kube.go:256] Updating pod openshift-ingress/router-default-58965f67cd-9ck8k I0403 00:03:34.623207 3870 kube.go:256] Updating pod openshift-console-operator/console-operator-57d9b95648-7swm2 I0403 00:03:34.633662 3870 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2] pod update took 22.667265ms I0403 00:03:34.633683 3870 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2] addLogicalPort annotation time took 22.691144ms I0403 00:03:34.634966 3870 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2] addLogicalPort took 24.068927ms, libovsdb time 946.971µs I0403 00:03:34.638574 3870 pod.go:62] [openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm] pod update took 22.662586ms I0403 00:03:34.638598 3870 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm] addLogicalPort annotation time took 22.688594ms I0403 00:03:34.639972 3870 pods.go:271] [openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm] addLogicalPort took 24.237929ms, libovsdb time 1.058729ms I0403 00:03:34.640031 3870 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-6465cb84c-r7q4t] creating logical port openshift-network-console_networking-console-plugin-6465cb84c-r7q4t for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.640134 3870 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-6465cb84c-r7q4t I0403 00:03:34.642384 3870 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-76c85c67b7-g4kf5] creating logical port openshift-ingress_router-default-76c85c67b7-g4kf5 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.642655 3870 kube.go:256] Updating pod openshift-ingress/router-default-76c85c67b7-g4kf5 I0403 00:03:34.642719 3870 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-5t4ss] creating logical port openshift-ingress-canary_ingress-canary-5t4ss for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.642837 3870 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-hxghc for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.642883 3870 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-5t4ss I0403 00:03:34.642977 3870 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc I0403 00:03:34.644974 3870 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q] pod update took 29.076962ms I0403 00:03:34.644991 3870 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q] addLogicalPort annotation time took 29.098575ms I0403 00:03:34.646299 3870 pod.go:62] [openshift-network-console/networking-console-plugin-6465cb84c-bmj99] pod update took 23.737919ms I0403 00:03:34.646317 3870 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-6465cb84c-bmj99] addLogicalPort annotation time took 23.764535ms I0403 00:03:34.646370 3870 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q] addLogicalPort took 30.581956ms, libovsdb time 1.08944ms I0403 00:03:34.647414 3870 pods.go:271] [openshift-network-console/networking-console-plugin-6465cb84c-bmj99] addLogicalPort took 24.928107ms, libovsdb time 784.838µs I0403 00:03:34.647763 3870 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-d4cbcbb94-gjwk7 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.647843 3870 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7 I0403 00:03:34.648184 3870 pod.go:62] [openshift-console-operator/console-operator-57d9b95648-7swm2] pod update took 25.006877ms I0403 00:03:34.648205 3870 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-57d9b95648-7swm2] addLogicalPort annotation time took 25.043266ms I0403 00:03:34.649413 3870 pods.go:271] [openshift-console-operator/console-operator-57d9b95648-7swm2] addLogicalPort took 26.31766ms, libovsdb time 909.126µs I0403 00:03:34.649468 3870 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc] creating logical port openshift-network-diagnostics_network-check-source-5bff8f5d77-4vzfc for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.649526 3870 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc I0403 00:03:34.651749 3870 pod.go:62] [openshift-insights/insights-operator-6f4dc9bddb-tv5j7] pod update took 28.905257ms I0403 00:03:34.651775 3870 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-6f4dc9bddb-tv5j7] addLogicalPort annotation time took 28.932632ms I0403 00:03:34.653448 3870 pods.go:271] [openshift-insights/insights-operator-6f4dc9bddb-tv5j7] addLogicalPort took 30.700301ms, libovsdb time 1.295899ms I0403 00:03:34.653510 3870 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc] creating logical port openshift-monitoring_cluster-monitoring-operator-65b957665b-cdpbc for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.653576 3870 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc I0403 00:03:34.655886 3870 pod.go:62] [openshift-ingress/router-default-58965f67cd-9ck8k] pod update took 32.74259ms I0403 00:03:34.655908 3870 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-58965f67cd-9ck8k] addLogicalPort annotation time took 32.764262ms I0403 00:03:34.657310 3870 pods.go:271] [openshift-ingress/router-default-58965f67cd-9ck8k] addLogicalPort took 34.277903ms, libovsdb time 1.116152ms I0403 00:03:34.657376 3870 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29586240-2xmxb for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.657457 3870 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb I0403 00:03:34.660160 3870 pod.go:62] [openshift-network-console/networking-console-plugin-6465cb84c-r7q4t] pod update took 20.049832ms I0403 00:03:34.660179 3870 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-6465cb84c-r7q4t] addLogicalPort annotation time took 20.074774ms I0403 00:03:34.661240 3870 pods.go:271] [openshift-network-console/networking-console-plugin-6465cb84c-r7q4t] addLogicalPort took 21.219189ms, libovsdb time 715.534µs I0403 00:03:34.662247 3870 pod.go:62] [openshift-ingress-canary/ingress-canary-5t4ss] pod update took 19.385609ms I0403 00:03:34.662267 3870 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-5t4ss] addLogicalPort annotation time took 19.406473ms I0403 00:03:34.663607 3870 pods.go:271] [openshift-ingress-canary/ingress-canary-5t4ss] addLogicalPort took 20.897162ms, libovsdb time 970.5µs I0403 00:03:34.664783 3870 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc] pod update took 21.862453ms I0403 00:03:34.664848 3870 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc] addLogicalPort annotation time took 21.928595ms I0403 00:03:34.666740 3870 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc] addLogicalPort took 23.920421ms, libovsdb time 1.443782ms I0403 00:03:34.667643 3870 pod.go:62] [openshift-ingress/router-default-76c85c67b7-g4kf5] pod update took 25.053442ms I0403 00:03:34.667707 3870 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-76c85c67b7-g4kf5] addLogicalPort annotation time took 25.120267ms I0403 00:03:34.668857 3870 pods.go:271] [openshift-ingress/router-default-76c85c67b7-g4kf5] addLogicalPort took 26.486709ms, libovsdb time 751.405µs I0403 00:03:34.670188 3870 pod.go:62] [openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc] pod update took 20.683281ms I0403 00:03:34.670210 3870 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc] addLogicalPort annotation time took 20.708754ms I0403 00:03:34.671530 3870 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc] pod update took 17.978176ms I0403 00:03:34.671541 3870 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7] pod update took 23.729197ms I0403 00:03:34.671546 3870 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc] addLogicalPort annotation time took 17.996346ms I0403 00:03:34.671554 3870 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7] addLogicalPort annotation time took 23.744213ms I0403 00:03:34.671585 3870 pods.go:271] [openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc] addLogicalPort took 22.123548ms, libovsdb time 1.047632ms I0403 00:03:34.671980 3870 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb] pod update took 14.547352ms I0403 00:03:34.672042 3870 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb] addLogicalPort annotation time took 14.610597ms I0403 00:03:34.672633 3870 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7] addLogicalPort took 24.876811ms, libovsdb time 757.991µs I0403 00:03:34.673040 3870 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc] addLogicalPort took 19.538392ms, libovsdb time 1.10131ms I0403 00:03:34.673416 3870 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb] addLogicalPort took 16.051588ms, libovsdb time 937.064µs I0403 00:03:34.703790 3870 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-trt2v] creating logical port openshift-dns_dns-default-trt2v for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:34.703901 3870 kube.go:256] Updating pod openshift-dns/dns-default-trt2v I0403 00:03:34.717378 3870 pod.go:62] [openshift-dns/dns-default-trt2v] pod update took 13.509521ms I0403 00:03:34.717406 3870 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-trt2v] addLogicalPort annotation time took 13.543159ms I0403 00:03:34.718682 3870 pods.go:271] [openshift-dns/dns-default-trt2v] addLogicalPort took 14.91046ms, libovsdb time 898.692µs I0403 00:03:34.943536 3870 cni.go:403] [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default] ADD starting CNI request [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default] I0403 00:03:34.966816 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-pruner-29586240-mslpc, hostIfaceName: b91b9655de35e08, network: default, NAD default, SandboxID: "b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31", PCI device ID: , UID: "6f37594e-3e0a-456a-a0e7-32ae9e83f21c", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0403 00:03:34.993551 3870 cni.go:403] [openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm b4f8e8daecb2ad8549edfd8465dfe76d783a89c869307f68e81d653501ff0210 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm b4f8e8daecb2ad8549edfd8465dfe76d783a89c869307f68e81d653501ff0210 network default NAD default] I0403 00:03:35.005910 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-54f5cf58db-5xbxm, hostIfaceName: b4f8e8daecb2ad8, network: default, NAD default, SandboxID: "b4f8e8daecb2ad8549edfd8465dfe76d783a89c869307f68e81d653501ff0210", PCI device ID: , UID: "b3d34bc8-fccd-45fc-a102-619803287ca6", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0403 00:03:35.006548 3870 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7 aa7c6099c5a1eb46d4988f9d87199c21344af53b653c218f70ad3788550d9792 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7 aa7c6099c5a1eb46d4988f9d87199c21344af53b653c218f70ad3788550d9792 network default NAD default] I0403 00:03:35.016743 3870 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q 4b8e626eea1abb204550273a41e11dc83571019b364be19506766d17152611b9 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q 4b8e626eea1abb204550273a41e11dc83571019b364be19506766d17152611b9 network default NAD default] I0403 00:03:35.019621 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-d4cbcbb94-gjwk7, hostIfaceName: aa7c6099c5a1eb4, network: default, NAD default, SandboxID: "aa7c6099c5a1eb46d4988f9d87199c21344af53b653c218f70ad3788550d9792", PCI device ID: , UID: "a951968e-bae1-493d-8f5b-cefc781fe228", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0403 00:03:35.032848 3870 cni.go:424] [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default] ADD finished CNI request [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:27:ae:92:c6:8a\",\"name\":\"b91b9655de35e08\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3dea5dbe-43dc-4bde-8f67-0737612a5092\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.036530 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-98c9d9db4-dgm4q, hostIfaceName: 4b8e626eea1abb2, network: default, NAD default, SandboxID: "4b8e626eea1abb204550273a41e11dc83571019b364be19506766d17152611b9", PCI device ID: , UID: "32ff8e73-e24f-4cf2-be5a-882f8aaa6fc2", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0403 00:03:35.061925 3870 cni.go:424] [openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm b4f8e8daecb2ad8549edfd8465dfe76d783a89c869307f68e81d653501ff0210 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-54f5cf58db-5xbxm b4f8e8daecb2ad8549edfd8465dfe76d783a89c869307f68e81d653501ff0210 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:06:39:c3:8c:58\",\"name\":\"b4f8e8daecb2ad8\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7fd66a33-1618-4682-b220-508f559d1798\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.070357 3870 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7 aa7c6099c5a1eb46d4988f9d87199c21344af53b653c218f70ad3788550d9792 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-gjwk7 aa7c6099c5a1eb46d4988f9d87199c21344af53b653c218f70ad3788550d9792 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:62:56:0c:ba:6f\",\"name\":\"aa7c6099c5a1eb4\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/243c01cb-eb6d-444c-97a8-aa2d9d13d5cc\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.084564 3870 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q 4b8e626eea1abb204550273a41e11dc83571019b364be19506766d17152611b9 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-dgm4q 4b8e626eea1abb204550273a41e11dc83571019b364be19506766d17152611b9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:d2:ff:bc:a6:6e\",\"name\":\"4b8e626eea1abb2\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8431544-d3ef-4990-8746-3ccf069cae54\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.192883 3870 cni.go:403] [openshift-ingress-canary/ingress-canary-5t4ss 49fb3dd5464fc6544cba2bc9444d51e97b2bf39f15e010451d07311d0d8ed37f network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-5t4ss 49fb3dd5464fc6544cba2bc9444d51e97b2bf39f15e010451d07311d0d8ed37f network default NAD default] I0403 00:03:35.194634 3870 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default] I0403 00:03:35.195529 3870 cni.go:403] [openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc 46542db686803b1b9fb13a5269d09baf3ca561e8f380128776e55072cbc2bb9e network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc 46542db686803b1b9fb13a5269d09baf3ca561e8f380128776e55072cbc2bb9e network default NAD default] I0403 00:03:35.195990 3870 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc cd8a739b066b960ede2f23dd0b73809c56cfc462a168d9653586dcd4beeb0213 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc cd8a739b066b960ede2f23dd0b73809c56cfc462a168d9653586dcd4beeb0213 network default NAD default] I0403 00:03:35.196141 3870 cni.go:403] [openshift-console-operator/console-operator-57d9b95648-7swm2 97c069f1c4fc7a2d61c2b91bf264b561da91caac2ae10d05f373eaf5e14c738d network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-57d9b95648-7swm2 97c069f1c4fc7a2d61c2b91bf264b561da91caac2ae10d05f373eaf5e14c738d network default NAD default] I0403 00:03:35.215599 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-5t4ss, hostIfaceName: 49fb3dd5464fc65, network: default, NAD default, SandboxID: "49fb3dd5464fc6544cba2bc9444d51e97b2bf39f15e010451d07311d0d8ed37f", PCI device ID: , UID: "8585cc6b-86b1-4f60-b8c4-4e2426c09e8c", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0403 00:03:35.215746 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29586240-2xmxb, hostIfaceName: 9f39427013697b4, network: default, NAD default, SandboxID: "9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18", PCI device ID: , UID: "19c3b0d5-00c9-494b-8e42-b04dab31c116", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0403 00:03:35.215998 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-hxghc, hostIfaceName: cd8a739b066b960, network: default, NAD default, SandboxID: "cd8a739b066b960ede2f23dd0b73809c56cfc462a168d9653586dcd4beeb0213", PCI device ID: , UID: "44ec066a-0e61-4f79-b0c8-2178aad1f58c", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0403 00:03:35.223500 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5bff8f5d77-4vzfc, hostIfaceName: 46542db686803b1, network: default, NAD default, SandboxID: "46542db686803b1b9fb13a5269d09baf3ca561e8f380128776e55072cbc2bb9e", PCI device ID: , UID: "45dfbfff-71b1-4e93-85a4-09fac80f6e87", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0403 00:03:35.224455 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-57d9b95648-7swm2, hostIfaceName: 97c069f1c4fc7a2, network: default, NAD default, SandboxID: "97c069f1c4fc7a2d61c2b91bf264b561da91caac2ae10d05f373eaf5e14c738d", PCI device ID: , UID: "af405712-54ff-4e8f-8c46-23b8aba15fbc", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0403 00:03:35.275736 3870 cni.go:424] [openshift-ingress-canary/ingress-canary-5t4ss 49fb3dd5464fc6544cba2bc9444d51e97b2bf39f15e010451d07311d0d8ed37f network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-5t4ss 49fb3dd5464fc6544cba2bc9444d51e97b2bf39f15e010451d07311d0d8ed37f network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:d8:cb:21:f7:b0\",\"name\":\"49fb3dd5464fc65\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7c358992-61d8-492b-8b08-55723ea51282\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.278752 3870 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc cd8a739b066b960ede2f23dd0b73809c56cfc462a168d9653586dcd4beeb0213 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-hxghc cd8a739b066b960ede2f23dd0b73809c56cfc462a168d9653586dcd4beeb0213 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:12:5d:0c:59:d9\",\"name\":\"cd8a739b066b960\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cfe9a4d7-0d19-43e6-b7b8-a5791bdd075f\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.283537 3870 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:99:ab:59:6f:5c\",\"name\":\"9f39427013697b4\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3063c739-2585-4592-819d-fd743ce96425\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.290835 3870 cni.go:424] [openshift-console-operator/console-operator-57d9b95648-7swm2 97c069f1c4fc7a2d61c2b91bf264b561da91caac2ae10d05f373eaf5e14c738d network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-57d9b95648-7swm2 97c069f1c4fc7a2d61c2b91bf264b561da91caac2ae10d05f373eaf5e14c738d network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:23:df:93:6c:7e\",\"name\":\"97c069f1c4fc7a2\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eada3e3d-243c-4a8e-b606-3a18626cdbb4\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.291207 3870 cni.go:424] [openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc 46542db686803b1b9fb13a5269d09baf3ca561e8f380128776e55072cbc2bb9e network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5bff8f5d77-4vzfc 46542db686803b1b9fb13a5269d09baf3ca561e8f380128776e55072cbc2bb9e network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:23:76:33:23:bc\",\"name\":\"46542db686803b1\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c105e561-134e-43ec-8bb1-9ac3100956da\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:35.370429 3870 cni.go:403] [openshift-insights/insights-operator-6f4dc9bddb-tv5j7 6dc4ce5605bd344e88ff3564ed100a2f61f0946e54d4feba51b6d5e7e2137510 network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-6f4dc9bddb-tv5j7 6dc4ce5605bd344e88ff3564ed100a2f61f0946e54d4feba51b6d5e7e2137510 network default NAD default] I0403 00:03:35.380968 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6f4dc9bddb-tv5j7, hostIfaceName: 6dc4ce5605bd344, network: default, NAD default, SandboxID: "6dc4ce5605bd344e88ff3564ed100a2f61f0946e54d4feba51b6d5e7e2137510", PCI device ID: , UID: "6f6942aa-28f1-4817-b56d-85f5b1f4bbaf", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0403 00:03:35.429104 3870 cni.go:424] [openshift-insights/insights-operator-6f4dc9bddb-tv5j7 6dc4ce5605bd344e88ff3564ed100a2f61f0946e54d4feba51b6d5e7e2137510 network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-6f4dc9bddb-tv5j7 6dc4ce5605bd344e88ff3564ed100a2f61f0946e54d4feba51b6d5e7e2137510 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:a7:48:ca:2d:06\",\"name\":\"6dc4ce5605bd344\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/28839aee-8e1c-4166-9a51-20fd0c0b8061\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:36.086826 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:03:36.086866 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 49.821µs I0403 00:03:36.086900 3870 egressfirewall.go:628] Syncing node "ip-10-0-1-110.ec2.internal" for egress firewall I0403 00:03:36.087077 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-110.ec2.internal in network "default" I0403 00:03:36.087104 3870 node_tracker.go:220] Node ip-10-0-1-110.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0403 00:03:36.087257 3870 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-110.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0403 00:03:36.087420 3870 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-110.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-110.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-110.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-110.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-110.ec2.internal, error parsing annotation for node ip-10-0-1-110.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0403 00:03:36.097592 3870 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-110.ec2.internal annotation: unexpected end of JSON input I0403 00:03:36.097628 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0403 00:03:36.097704 3870 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-110.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-110.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-110.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-110.ec2.internal" E0403 00:03:36.097869 3870 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-110.ec2.internal annotation: unexpected end of JSON input I0403 00:03:36.107793 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0403 00:03:36.112496 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-147.ec2.internal I0403 00:03:36.112568 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-147.ec2.internal: 76.909µs I0403 00:03:36.112583 3870 egressfirewall.go:628] Syncing node "ip-10-0-0-147.ec2.internal" for egress firewall I0403 00:03:36.112630 3870 default_network_controller.go:1008] Node "ip-10-0-0-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:36.124000 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:36.143214 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:03:36.143245 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 37.85µs I0403 00:03:36.143335 3870 egressfirewall.go:628] Syncing node "ip-10-0-1-110.ec2.internal" for egress firewall I0403 00:03:36.143642 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:36.346539 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:03:36.346567 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 36.036µs I0403 00:03:36.346700 3870 egressfirewall.go:628] Syncing node "ip-10-0-1-110.ec2.internal" for egress firewall I0403 00:03:36.346955 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:36.370709 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:36.619539 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:36.943372 3870 cni.go:403] [openshift-network-diagnostics/network-check-target-zgkc5 9b7210aea673fa5b5b9fbaf7f6ce22edb3378b4bfe1bc8f29c936d296b9264ed network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-zgkc5 9b7210aea673fa5b5b9fbaf7f6ce22edb3378b4bfe1bc8f29c936d296b9264ed network default NAD default] I0403 00:03:36.955027 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-zgkc5, hostIfaceName: 9b7210aea673fa5, network: default, NAD default, SandboxID: "9b7210aea673fa5b5b9fbaf7f6ce22edb3378b4bfe1bc8f29c936d296b9264ed", PCI device ID: , UID: "3968bf26-04f5-45f2-a1b3-f2f8e856302d", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0403 00:03:37.015684 3870 cni.go:424] [openshift-network-diagnostics/network-check-target-zgkc5 9b7210aea673fa5b5b9fbaf7f6ce22edb3378b4bfe1bc8f29c936d296b9264ed network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-zgkc5 9b7210aea673fa5b5b9fbaf7f6ce22edb3378b4bfe1bc8f29c936d296b9264ed network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:1c:8f:16:9e:a1\",\"name\":\"9b7210aea673fa5\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9af75367-4799-45b8-8b3b-e5f8caa36ed1\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:37.114625 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-147.ec2.internal I0403 00:03:37.114635 3870 egressfirewall.go:628] Syncing node "ip-10-0-0-147.ec2.internal" for egress firewall I0403 00:03:37.114649 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-147.ec2.internal: 30.452µs I0403 00:03:37.114729 3870 default_network_controller.go:1008] Node "ip-10-0-0-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:40.387916 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:41.892206 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-147.ec2.internal in network "default" I0403 00:03:41.892259 3870 node_tracker.go:235] Node ip-10-0-0-147.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-147.ec2.internal" I0403 00:03:41.892288 3870 node_tracker.go:169] Node ip-10-0-0-147.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:41.892275 3870 default_network_controller.go:1008] Node "ip-10-0-0-147.ec2.internal" in remote zone "ip-10-0-0-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:41.892302 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:41.975493 3870 default_network_controller.go:1008] Node "ip-10-0-0-147.ec2.internal" in remote zone "ip-10-0-0-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:42.015687 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-147.ec2.internal in network "default" I0403 00:03:42.015757 3870 node_tracker.go:169] Node ip-10-0-0-147.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:42.015773 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:42.015769 3870 default_network_controller.go:1008] Node "ip-10-0-0-147.ec2.internal" in remote zone "ip-10-0-0-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:42.016457 3870 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-147.ec2.internal for the network default I0403 00:03:42.017847 3870 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-147.ec2.internal" on network "default" took: 1.401861ms I0403 00:03:42.837632 3870 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-nznpj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:03:42.837678 3870 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-nznpj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0403 00:03:43.544673 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0403 00:03:43.556573 3870 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-nznpj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:03:43.556598 3870 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-nznpj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0403 00:03:43.655344 3870 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0403 00:03:43.758340 3870 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:03:43.767442 3870 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29586240-nt956 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:03:43.767475 3870 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29586240-nt956 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:03:44.070309 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-147.ec2.internal in network "default" I0403 00:03:44.070341 3870 egressfirewall.go:628] Syncing node "ip-10-0-0-147.ec2.internal" for egress firewall I0403 00:03:44.070388 3870 node_tracker.go:169] Node ip-10-0-0-147.ec2.internal switch + router changed, syncing services in network "default" I0403 00:03:44.070389 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-147.ec2.internal", IP: 10.0.0.147 to PMTUD blocking rules I0403 00:03:44.070402 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-147.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0403 00:03:44.070405 3870 services_controller.go:574] Full service sync requested for network=default I0403 00:03:44.070319 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-147.ec2.internal I0403 00:03:44.070447 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-147.ec2.internal: 133.318µs W0403 00:03:44.070684 3870 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0403 00:03:44.070798 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0403 00:03:44.070897 3870 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:03:44.076497 3870 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-147.ec2.internal. Updating no re-route policies I0403 00:03:44.089125 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-147.ec2.internal", IP: 10.0.0.147 to PMTUD blocking rules I0403 00:03:44.089195 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-147.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0403 00:03:44.238483 3870 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0403 00:03:44.239419 3870 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 912.753µs I0403 00:03:44.256976 3870 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0403 00:03:44.347405 3870 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q] creating logical port openshift-kube-storage-version-migrator_migrator-848b55fff-cqd2q for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:44.347593 3870 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q I0403 00:03:44.364171 3870 pod.go:62] [openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q] pod update took 16.618254ms I0403 00:03:44.364192 3870 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q] addLogicalPort annotation time took 16.646196ms I0403 00:03:44.365690 3870 pods.go:271] [openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q] addLogicalPort took 18.318118ms, libovsdb time 1.052714ms I0403 00:03:44.564078 3870 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29586240-nt956 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:03:44.564103 3870 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29586240-nt956 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:03:44.757747 3870 cni.go:403] [openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q 0eef446f05a54dd1c6ba51446b2fa339963d46c8059da0c73da57de73b33ec04 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q 0eef446f05a54dd1c6ba51446b2fa339963d46c8059da0c73da57de73b33ec04 network default NAD default] I0403 00:03:44.786764 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-848b55fff-cqd2q, hostIfaceName: 0eef446f05a54dd, network: default, NAD default, SandboxID: "0eef446f05a54dd1c6ba51446b2fa339963d46c8059da0c73da57de73b33ec04", PCI device ID: , UID: "140024b0-eb4f-42a7-8944-eb72c9b9b848", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0403 00:03:44.833858 3870 cni.go:424] [openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q 0eef446f05a54dd1c6ba51446b2fa339963d46c8059da0c73da57de73b33ec04 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-848b55fff-cqd2q 0eef446f05a54dd1c6ba51446b2fa339963d46c8059da0c73da57de73b33ec04 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:1d:5b:39:b6:ef\",\"name\":\"0eef446f05a54dd\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36d93872-25a9-4396-aee4-d53f8b3b9afc\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:45.285681 3870 namespace.go:98] [openshift-service-ca] adding namespace I0403 00:03:45.286603 3870 namespace.go:102] [openshift-service-ca] adding namespace took 893.218µs I0403 00:03:45.301361 3870 namespace.go:142] [openshift-service-ca] updating namespace I0403 00:03:46.302626 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:46.959155 3870 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-b95c4699b-zcz67] creating logical port openshift-service-ca_service-ca-b95c4699b-zcz67 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:46.959306 3870 kube.go:256] Updating pod openshift-service-ca/service-ca-b95c4699b-zcz67 I0403 00:03:46.970611 3870 pod.go:62] [openshift-service-ca/service-ca-b95c4699b-zcz67] pod update took 11.337021ms I0403 00:03:46.970633 3870 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-b95c4699b-zcz67] addLogicalPort annotation time took 11.364951ms I0403 00:03:46.971880 3870 pods.go:271] [openshift-service-ca/service-ca-b95c4699b-zcz67] addLogicalPort took 12.746615ms, libovsdb time 914.746µs I0403 00:03:47.346103 3870 cni.go:403] [openshift-service-ca/service-ca-b95c4699b-zcz67 83df8d43dfbc0df89fedc0a3920efcc4d3f4d3c4e0821b71f11c939eacc27ebd network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-b95c4699b-zcz67 83df8d43dfbc0df89fedc0a3920efcc4d3f4d3c4e0821b71f11c939eacc27ebd network default NAD default] I0403 00:03:47.355251 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-b95c4699b-zcz67, hostIfaceName: 83df8d43dfbc0df, network: default, NAD default, SandboxID: "83df8d43dfbc0df89fedc0a3920efcc4d3f4d3c4e0821b71f11c939eacc27ebd", PCI device ID: , UID: "b56427e2-b4cc-4d43-bb33-b9922bbaab6a", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0403 00:03:47.402142 3870 cni.go:424] [openshift-service-ca/service-ca-b95c4699b-zcz67 83df8d43dfbc0df89fedc0a3920efcc4d3f4d3c4e0821b71f11c939eacc27ebd network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-b95c4699b-zcz67 83df8d43dfbc0df89fedc0a3920efcc4d3f4d3c4e0821b71f11c939eacc27ebd network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f9:85:97:05:7d\",\"name\":\"83df8d43dfbc0df\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b669f9c4-5254-4b87-b7ec-2d622d565d97\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:47.930830 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-147.ec2.internal I0403 00:03:47.930855 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-147.ec2.internal: 32.002µs I0403 00:03:48.414269 3870 cni.go:403] [kube-system/global-pull-secret-syncer-nxhhs e72bb62668c3d0072976fa5ae323503be40b3ac2599914a603b68be10bd926d7 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-nxhhs e72bb62668c3d0072976fa5ae323503be40b3ac2599914a603b68be10bd926d7 network default NAD default] I0403 00:03:48.424371 3870 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-nxhhs, hostIfaceName: e72bb62668c3d00, network: default, NAD default, SandboxID: "e72bb62668c3d0072976fa5ae323503be40b3ac2599914a603b68be10bd926d7", PCI device ID: , UID: "b35dc69d-d9f8-4809-a544-e8f299aef387", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0403 00:03:48.472630 3870 cni.go:424] [kube-system/global-pull-secret-syncer-nxhhs e72bb62668c3d0072976fa5ae323503be40b3ac2599914a603b68be10bd926d7 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-nxhhs e72bb62668c3d0072976fa5ae323503be40b3ac2599914a603b68be10bd926d7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:84:91:c8:7b:1c\",\"name\":\"e72bb62668c3d00\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0bb54616-c11e-4fea-a51d-c175c215b663\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:50.586220 3870 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2 56312280d2111285f2453111fdf319a840e2c8b62a99e9d769698d694f98a29c network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2 56312280d2111285f2453111fdf319a840e2c8b62a99e9d769698d694f98a29c network default NAD default] I0403 00:03:50.595947 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-75b48889dc-z74x2, hostIfaceName: 56312280d211128, network: default, NAD default, SandboxID: "56312280d2111285f2453111fdf319a840e2c8b62a99e9d769698d694f98a29c", PCI device ID: , UID: "0329f0f4-0d46-4c6b-8190-d017996cd526", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0403 00:03:50.645836 3870 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2 56312280d2111285f2453111fdf319a840e2c8b62a99e9d769698d694f98a29c network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-z74x2 56312280d2111285f2453111fdf319a840e2c8b62a99e9d769698d694f98a29c network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:f8:b9:f5:13:e3\",\"name\":\"56312280d211128\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6ac5da0e-d9a1-44f3-a81c-8c17da56e4d5\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:03:50.915894 3870 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-m2t92] creating logical port openshift-insights_insights-runtime-extractor-m2t92 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:03:50.916014 3870 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-m2t92 I0403 00:03:50.934039 3870 pod.go:62] [openshift-insights/insights-runtime-extractor-m2t92] pod update took 18.054775ms I0403 00:03:50.934081 3870 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-m2t92] addLogicalPort annotation time took 18.088055ms I0403 00:03:50.937376 3870 pods.go:271] [openshift-insights/insights-runtime-extractor-m2t92] addLogicalPort took 21.496149ms, libovsdb time 2.918373ms W0403 00:03:52.644531 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0403 00:03:58.011826 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:03:58.011842 3870 egressfirewall.go:628] Syncing node "ip-10-0-1-110.ec2.internal" for egress firewall I0403 00:03:58.011842 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:03:58.011847 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 29.439µs I0403 00:04:00.942480 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nfwxx I0403 00:04:00.942512 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nfwxx I0403 00:04:00.942550 3870 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-nfwxx, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nfwxx: suppressed error logged: pod openshift-multus/network-metrics-daemon-nfwxx: no pod IPs found I0403 00:04:00.942483 3870 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-swclw I0403 00:04:00.942561 3870 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-swclw I0403 00:04:00.942580 3870 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-swclw, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-swclw: suppressed error logged: pod openshift-network-diagnostics/network-check-target-swclw: no pod IPs found I0403 00:04:06.718935 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:04:06.770529 3870 cni.go:403] [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default] I0403 00:04:06.779656 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7c687859fb-z2cst, hostIfaceName: a108ce29037c159, network: default, NAD default, SandboxID: "a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5", PCI device ID: , UID: "53b0b485-1b5c-442c-8468-cc96509a4026", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0403 00:04:06.829170 3870 cni.go:403] [openshift-network-console/networking-console-plugin-6465cb84c-bmj99 6a8ae32ff92a4e587ae24deea8d6cef9e9dc5da9a1d8e83cfffa75f7930bb042 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-6465cb84c-bmj99 6a8ae32ff92a4e587ae24deea8d6cef9e9dc5da9a1d8e83cfffa75f7930bb042 network default NAD default] I0403 00:04:06.834384 3870 cni.go:424] [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:de:58:18:e3:1f\",\"name\":\"a108ce29037c159\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/97e6cbb4-07bc-45f7-b13d-a1c8ad2b95f7\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:06.839907 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-6465cb84c-bmj99, hostIfaceName: 6a8ae32ff92a4e5, network: default, NAD default, SandboxID: "6a8ae32ff92a4e587ae24deea8d6cef9e9dc5da9a1d8e83cfffa75f7930bb042", PCI device ID: , UID: "5184718e-b177-4140-8ceb-97e2105bfa06", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0403 00:04:06.867137 3870 cni.go:403] [openshift-network-console/networking-console-plugin-6465cb84c-r7q4t 3d904186ebd3b8535b505700457696430b8156e05530a12ef4fba8cd32894e5c network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-6465cb84c-r7q4t 3d904186ebd3b8535b505700457696430b8156e05530a12ef4fba8cd32894e5c network default NAD default] I0403 00:04:06.873717 3870 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc e60b63382f65031a6c7832ed9d8bd5fafa16b92750cfaf64faa056723b39fb82 network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc e60b63382f65031a6c7832ed9d8bd5fafa16b92750cfaf64faa056723b39fb82 network default NAD default] I0403 00:04:06.878584 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-6465cb84c-r7q4t, hostIfaceName: 3d904186ebd3b85, network: default, NAD default, SandboxID: "3d904186ebd3b8535b505700457696430b8156e05530a12ef4fba8cd32894e5c", PCI device ID: , UID: "67418994-5339-4218-b3c3-55f928c5e97e", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0403 00:04:06.878881 3870 cni.go:403] [openshift-ingress/router-default-58965f67cd-9ck8k 73a8662d189760f8053147fe44be31088e5a2090d199250b0f232fea8b4ead52 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-58965f67cd-9ck8k 73a8662d189760f8053147fe44be31088e5a2090d199250b0f232fea8b4ead52 network default NAD default] I0403 00:04:06.882378 3870 cni.go:403] [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default] ADD starting CNI request [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default] I0403 00:04:06.896055 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-76c85c67b7-g4kf5, hostIfaceName: d620160f287a733, network: default, NAD default, SandboxID: "d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc", PCI device ID: , UID: "5160d1d0-2aaa-4623-b7ab-1e7446873d60", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0403 00:04:06.897364 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-65b957665b-cdpbc, hostIfaceName: e60b63382f65031, network: default, NAD default, SandboxID: "e60b63382f65031a6c7832ed9d8bd5fafa16b92750cfaf64faa056723b39fb82", PCI device ID: , UID: "b1636345-8c73-476a-9103-f7057fdeee1e", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0403 00:04:06.898931 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-58965f67cd-9ck8k, hostIfaceName: 73a8662d189760f, network: default, NAD default, SandboxID: "73a8662d189760f8053147fe44be31088e5a2090d199250b0f232fea8b4ead52", PCI device ID: , UID: "dd4cda7c-0910-41c1-a9db-81e5b0bdb9d8", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0403 00:04:06.902570 3870 cni.go:424] [openshift-network-console/networking-console-plugin-6465cb84c-bmj99 6a8ae32ff92a4e587ae24deea8d6cef9e9dc5da9a1d8e83cfffa75f7930bb042 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-6465cb84c-bmj99 6a8ae32ff92a4e587ae24deea8d6cef9e9dc5da9a1d8e83cfffa75f7930bb042 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:b5:a7:62:15:76\",\"name\":\"6a8ae32ff92a4e5\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5056546c-4430-44ad-ac35-c948fead0254\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:06.930782 3870 cni.go:403] [openshift-dns/dns-default-trt2v 485191e596e6b4d9ac9c4db2991651b5bf511400cc7b833387dc75f58eb62138 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-trt2v 485191e596e6b4d9ac9c4db2991651b5bf511400cc7b833387dc75f58eb62138 network default NAD default] I0403 00:04:06.936499 3870 cni.go:403] [openshift-insights/insights-runtime-extractor-m2t92 21b27575447adf3957b3a7fcb8c96318311b070cc186ec5f2f525fcda244e2f7 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-m2t92 21b27575447adf3957b3a7fcb8c96318311b070cc186ec5f2f525fcda244e2f7 network default NAD default] I0403 00:04:06.946529 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-trt2v, hostIfaceName: 485191e596e6b4d, network: default, NAD default, SandboxID: "485191e596e6b4d9ac9c4db2991651b5bf511400cc7b833387dc75f58eb62138", PCI device ID: , UID: "c20b8023-bcaa-4704-a490-52e14650c075", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0403 00:04:06.947366 3870 cni.go:424] [openshift-network-console/networking-console-plugin-6465cb84c-r7q4t 3d904186ebd3b8535b505700457696430b8156e05530a12ef4fba8cd32894e5c network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-6465cb84c-r7q4t 3d904186ebd3b8535b505700457696430b8156e05530a12ef4fba8cd32894e5c network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:33:23:95:16:2e\",\"name\":\"3d904186ebd3b85\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9c85748f-10c4-4966-8ef7-77440937c8c4\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:06.950515 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-m2t92, hostIfaceName: 21b27575447adf3, network: default, NAD default, SandboxID: "21b27575447adf3957b3a7fcb8c96318311b070cc186ec5f2f525fcda244e2f7", PCI device ID: , UID: "b24fe6b2-4a7e-4d2e-b6af-9ad10af8878e", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0403 00:04:07.172829 3870 cni.go:424] [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default] ADD finished CNI request [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:33:09:19:7f:bf\",\"name\":\"d620160f287a733\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/faa484ac-2df4-4541-bd40-99678fa3ffda\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:07.174633 3870 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc e60b63382f65031a6c7832ed9d8bd5fafa16b92750cfaf64faa056723b39fb82 network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-65b957665b-cdpbc e60b63382f65031a6c7832ed9d8bd5fafa16b92750cfaf64faa056723b39fb82 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:73:79:45:63:57\",\"name\":\"e60b63382f65031\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65500b12-f976-4e12-9296-023010d4c6c0\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:07.176981 3870 cni.go:424] [openshift-ingress/router-default-58965f67cd-9ck8k 73a8662d189760f8053147fe44be31088e5a2090d199250b0f232fea8b4ead52 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-58965f67cd-9ck8k 73a8662d189760f8053147fe44be31088e5a2090d199250b0f232fea8b4ead52 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:ba:39:e5:ed:db\",\"name\":\"73a8662d189760f\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd03a80b-caa9-49f5-9958-12a089bc62eb\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:07.224234 3870 cni.go:424] [openshift-dns/dns-default-trt2v 485191e596e6b4d9ac9c4db2991651b5bf511400cc7b833387dc75f58eb62138 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-trt2v 485191e596e6b4d9ac9c4db2991651b5bf511400cc7b833387dc75f58eb62138 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:ca:21:23:ae:f9\",\"name\":\"485191e596e6b4d\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0521f1f-27e4-4f61-8dc2-6371734622aa\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:07.227599 3870 cni.go:424] [openshift-insights/insights-runtime-extractor-m2t92 21b27575447adf3957b3a7fcb8c96318311b070cc186ec5f2f525fcda244e2f7 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-m2t92 21b27575447adf3957b3a7fcb8c96318311b070cc186ec5f2f525fcda244e2f7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:c2:b6:f2:dd:3c\",\"name\":\"21b27575447adf3\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4c6c1a4-f27a-45e4-92d1-fb9fe017801e\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:08.685167 3870 cni.go:403] [openshift-multus/network-metrics-daemon-9xsl8 c8a3a8c66bb5b4274330361a236dcfbcd2af2886df43fd5be82ec1b1cf30a880 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-9xsl8 c8a3a8c66bb5b4274330361a236dcfbcd2af2886df43fd5be82ec1b1cf30a880 network default NAD default] I0403 00:04:08.695523 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-9xsl8, hostIfaceName: c8a3a8c66bb5b42, network: default, NAD default, SandboxID: "c8a3a8c66bb5b4274330361a236dcfbcd2af2886df43fd5be82ec1b1cf30a880", PCI device ID: , UID: "e7e2f9f9-7db3-4f61-9386-0c7cc34240a2", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0403 00:04:08.745819 3870 cni.go:424] [openshift-multus/network-metrics-daemon-9xsl8 c8a3a8c66bb5b4274330361a236dcfbcd2af2886df43fd5be82ec1b1cf30a880 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-9xsl8 c8a3a8c66bb5b4274330361a236dcfbcd2af2886df43fd5be82ec1b1cf30a880 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:44:0e:b6:7c:f8\",\"name\":\"c8a3a8c66bb5b42\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fad4d5ae-a142-4087-bcb3-ee100498591b\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:04:11.634442 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0403 00:04:11.645909 3870 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0403 00:04:11.676321 3870 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:11.676342 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:04:11.776743 3870 base_network_controller_pods.go:487] [default/openshift-console/downloads-744b9bb959-58clx] creating logical port openshift-console_downloads-744b9bb959-58clx for pod on switch ip-10-0-2-108.ec2.internal I0403 00:04:11.776845 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:04:11.776915 3870 kube.go:256] Updating pod openshift-console/downloads-744b9bb959-58clx I0403 00:04:11.781101 3870 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-7464b9f894-s4gfb for pod on switch ip-10-0-2-108.ec2.internal I0403 00:04:11.781230 3870 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb I0403 00:04:11.798089 3870 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0403 00:04:11.811792 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:04:11.811812 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0403 00:04:11.811817 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:04:11.815122 3870 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-5d6c49ff-htht4 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-5d6c49ff-htht4: no pod IPs found I0403 00:04:11.825366 3870 pod.go:62] [openshift-console/downloads-744b9bb959-58clx] pod update took 48.480742ms I0403 00:04:11.825390 3870 base_network_controller_pods.go:937] [default/openshift-console/downloads-744b9bb959-58clx] addLogicalPort annotation time took 48.512879ms I0403 00:04:11.826678 3870 pods.go:271] [openshift-console/downloads-744b9bb959-58clx] addLogicalPort took 49.954219ms, libovsdb time 885.39µs I0403 00:04:11.844719 3870 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb] pod update took 63.514913ms I0403 00:04:11.844742 3870 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb] addLogicalPort annotation time took 63.544263ms I0403 00:04:11.846051 3870 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb] addLogicalPort took 64.966848ms, libovsdb time 910.87µs I0403 00:04:11.846127 3870 base_network_controller_pods.go:487] [default/openshift-console/console-56dbfb7889-fb65s] creating logical port openshift-console_console-56dbfb7889-fb65s for pod on switch ip-10-0-2-108.ec2.internal I0403 00:04:11.846212 3870 kube.go:256] Updating pod openshift-console/console-56dbfb7889-fb65s W0403 00:04:11.897259 3870 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0403 00:04:11.921446 3870 kube.go:256] Updating pod openshift-console/console-56dbfb7889-fb65s I0403 00:04:11.984470 3870 pod.go:62] [openshift-console/console-56dbfb7889-fb65s] pod update took 138.280628ms I0403 00:04:11.984498 3870 base_network_controller_pods.go:937] [default/openshift-console/console-56dbfb7889-fb65s] addLogicalPort annotation time took 138.314156ms I0403 00:04:11.985674 3870 pods.go:271] [openshift-console/console-56dbfb7889-fb65s] addLogicalPort took 139.552434ms, libovsdb time 832.62µs I0403 00:04:11.988500 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:04:12.174291 3870 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb 317603b856f2942e561c66740ee6e5467c5066336119360634ca2120da70bf51 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb 317603b856f2942e561c66740ee6e5467c5066336119360634ca2120da70bf51 network default NAD default] I0403 00:04:12.184550 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-7464b9f894-s4gfb, hostIfaceName: 317603b856f2942, network: default, NAD default, SandboxID: "317603b856f2942e561c66740ee6e5467c5066336119360634ca2120da70bf51", PCI device ID: , UID: "357d89e6-b896-48ba-bdd8-8478093a2c79", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0403 00:04:12.202192 3870 cni.go:403] [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default] ADD starting CNI request [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default] I0403 00:04:12.213570 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-56dbfb7889-fb65s, hostIfaceName: 0d804d383797ace, network: default, NAD default, SandboxID: "0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13", PCI device ID: , UID: "3dae658e-733e-482d-9be3-dcfb9b5b0b71", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0403 00:04:12.241292 3870 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb 317603b856f2942e561c66740ee6e5467c5066336119360634ca2120da70bf51 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-s4gfb 317603b856f2942e561c66740ee6e5467c5066336119360634ca2120da70bf51 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:ae:7f:3e:a5:36\",\"name\":\"317603b856f2942\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36b5c3c3-2bbc-4a0f-b4e5-8c55876d80e5\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:12.263554 3870 cni.go:424] [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default] ADD finished CNI request [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:57:82:b3:8a:b5\",\"name\":\"0d804d383797ace\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/125ecf69-c73c-437c-884e-b69365c172ce\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:12.324876 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "ip-10-0-1-110.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:04:12.324913 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-110.ec2.internal in network "default" I0403 00:04:12.325007 3870 node_tracker.go:235] Node ip-10-0-1-110.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-110.ec2.internal" I0403 00:04:12.325028 3870 node_tracker.go:169] Node ip-10-0-1-110.ec2.internal switch + router changed, syncing services in network "default" I0403 00:04:12.325042 3870 services_controller.go:574] Full service sync requested for network=default W0403 00:04:12.325495 3870 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:12.325509 3870 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:04:12.450911 3870 cni.go:403] [openshift-console/downloads-744b9bb959-58clx 47b63eedff9c9e947fdf5ffc252692e79144a7ca2bbe69fc0bfc8f462807d954 network default NAD default] ADD starting CNI request [openshift-console/downloads-744b9bb959-58clx 47b63eedff9c9e947fdf5ffc252692e79144a7ca2bbe69fc0bfc8f462807d954 network default NAD default] I0403 00:04:12.460275 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-744b9bb959-58clx, hostIfaceName: 47b63eedff9c9e9, network: default, NAD default, SandboxID: "47b63eedff9c9e947fdf5ffc252692e79144a7ca2bbe69fc0bfc8f462807d954", PCI device ID: , UID: "28748d66-c325-4eac-aa51-86edde7ca9b6", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0403 00:04:12.462513 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-110.ec2.internal in network "default" I0403 00:04:12.462564 3870 node_tracker.go:169] Node ip-10-0-1-110.ec2.internal switch + router changed, syncing services in network "default" I0403 00:04:12.462560 3870 default_network_controller.go:1008] Node "ip-10-0-1-110.ec2.internal" in remote zone "ip-10-0-1-110.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:04:12.462579 3870 services_controller.go:574] Full service sync requested for network=default W0403 00:04:12.462665 3870 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:12.462718 3870 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:04:12.463281 3870 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-110.ec2.internal for the network default I0403 00:04:12.464463 3870 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-110.ec2.internal" on network "default" took: 1.198618ms I0403 00:04:12.719497 3870 cni.go:424] [openshift-console/downloads-744b9bb959-58clx 47b63eedff9c9e947fdf5ffc252692e79144a7ca2bbe69fc0bfc8f462807d954 network default NAD default] ADD finished CNI request [openshift-console/downloads-744b9bb959-58clx 47b63eedff9c9e947fdf5ffc252692e79144a7ca2bbe69fc0bfc8f462807d954 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:82:46:22:22:75\",\"name\":\"47b63eedff9c9e9\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c216fb6e-fbb2-4a19-b347-30f3143e8742\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:13.020157 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:04:13.020175 3870 egressfirewall.go:628] Syncing node "ip-10-0-1-110.ec2.internal" for egress firewall I0403 00:04:13.020186 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 36.871µs I0403 00:04:13.020164 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-110.ec2.internal", IP: 10.0.1.110 to PMTUD blocking rules I0403 00:04:13.020202 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-110.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0403 00:04:13.020188 3870 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-110.ec2.internal in network "default" I0403 00:04:13.020328 3870 node_tracker.go:169] Node ip-10-0-1-110.ec2.internal switch + router changed, syncing services in network "default" I0403 00:04:13.020347 3870 services_controller.go:574] Full service sync requested for network=default W0403 00:04:13.020730 3870 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:13.020769 3870 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:04:13.026275 3870 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-110.ec2.internal. Updating no re-route policies I0403 00:04:13.048938 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-110.ec2.internal", IP: 10.0.1.110 to PMTUD blocking rules I0403 00:04:13.048958 3870 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-110.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0403 00:04:14.560526 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0403 00:04:14.713181 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0403 00:04:14.736368 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0403 00:04:16.375046 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0403 00:04:16.723811 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0403 00:04:16.752181 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:04:16.752211 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 38.639µs W0403 00:04:18.166718 3870 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:21.714328 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:04:21.714351 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:04:21.714357 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0403 00:04:21.826194 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:21.826218 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:22.209853 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0403 00:04:23.413051 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:23.413096 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:27.215821 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:27.215848 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:27.253369 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:27.253400 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:28.686983 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:04:29.783338 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0403 00:04:29.805204 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0403 00:04:35.201086 3870 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0403 00:04:36.284199 3870 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:04:36.284225 3870 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:04:36.284238 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:04:37.445344 3870 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-86d9fd7876-cpmgs IP address from the namespace address-set, err: pod openshift-console/console-86d9fd7876-cpmgs: no pod IPs found I0403 00:04:38.841162 3870 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c687859fb-28dsx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:38.841724 3870 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c687859fb-28dsx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0403 00:04:39.546366 3870 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:04:39.546385 3870 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:04:39.546390 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0403 00:04:39.593600 3870 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c687859fb-28dsx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:39.593623 3870 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c687859fb-28dsx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:39.605160 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c687859fb-28dsx I0403 00:04:39.605178 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c687859fb-28dsx W0403 00:04:40.289945 3870 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:04:40.289964 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:04:40.535439 3870 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:04:40.535459 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0403 00:04:41.678272 3870 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:41.678863 3870 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:41.678888 3870 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:04:41.678899 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0403 00:04:41.678917 3870 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:04:41.678923 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0403 00:04:42.280333 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0403 00:04:42.579211 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0403 00:04:42.704927 3870 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:42.704948 3870 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:42.704961 3870 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:42.704971 3870 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539026mpc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:42.885583 3870 cni.go:403] [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default] DEL starting CNI request [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default] I0403 00:04:42.965986 3870 cni.go:424] [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default] DEL finished CNI request [openshift-console/console-56dbfb7889-fb65s 0d804d383797ace43c57958f6d399236c15ba4ceee90bce03199090cbb855d13 network default NAD default], result "{}", err I0403 00:04:43.039321 3870 obj_retry.go:464] Detected object openshift-console/console-56dbfb7889-fb65s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:43.039361 3870 pods.go:176] Deleting pod: openshift-console/console-56dbfb7889-fb65s I0403 00:04:43.041147 3870 pods.go:236] Attempting to release IPs for pod: openshift-console/console-56dbfb7889-fb65s, ips: 10.128.0.29 I0403 00:04:43.041178 3870 obj_retry.go:464] Detected object openshift-console/console-56dbfb7889-fb65s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:43.049418 3870 base_network_controller_pods.go:487] [default/openshift-console/console-db7994b7c-6lh5g] creating logical port openshift-console_console-db7994b7c-6lh5g for pod on switch ip-10-0-2-108.ec2.internal I0403 00:04:43.049531 3870 kube.go:256] Updating pod openshift-console/console-db7994b7c-6lh5g I0403 00:04:43.065950 3870 pod.go:62] [openshift-console/console-db7994b7c-6lh5g] pod update took 16.44431ms I0403 00:04:43.065973 3870 base_network_controller_pods.go:937] [default/openshift-console/console-db7994b7c-6lh5g] addLogicalPort annotation time took 16.472278ms I0403 00:04:43.067462 3870 pods.go:271] [openshift-console/console-db7994b7c-6lh5g] addLogicalPort took 18.055202ms, libovsdb time 1.099924ms W0403 00:04:43.346445 3870 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:04:43.346469 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:04:43.346475 3870 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:04:43.346478 3870 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0403 00:04:43.447722 3870 cni.go:403] [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default] ADD starting CNI request [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default] I0403 00:04:43.457281 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-db7994b7c-6lh5g, hostIfaceName: 166e96df0bf9f55, network: default, NAD default, SandboxID: "166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f", PCI device ID: , UID: "4df470fd-0b09-4668-92b7-af6a6683fbae", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0403 00:04:43.507592 3870 cni.go:424] [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default] ADD finished CNI request [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:45:da:b3:49:d5\",\"name\":\"166e96df0bf9f55\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/34a35972-fa10-4c3d-9441-5977519d23a4\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:04:43.876525 3870 obj_retry.go:458] Detected object openshift-console/console-56dbfb7889-fb65s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:43.876546 3870 obj_retry.go:458] Detected object openshift-console/console-56dbfb7889-fb65s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:43.879299 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56dbfb7889-fb65s I0403 00:04:43.879326 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56dbfb7889-fb65s W0403 00:04:44.312703 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0403 00:04:44.647683 3870 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0403 00:04:44.663421 3870 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0403 00:04:52.549348 3870 obj_retry.go:464] Detected object openshift-console/console-56dbfb7889-njmjr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:52.549861 3870 obj_retry.go:464] Detected object openshift-console/console-56dbfb7889-njmjr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:04:52.886454 3870 obj_retry.go:458] Detected object openshift-console/console-56dbfb7889-njmjr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:52.886477 3870 obj_retry.go:458] Detected object openshift-console/console-56dbfb7889-njmjr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:04:52.889375 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56dbfb7889-njmjr I0403 00:04:52.889399 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56dbfb7889-njmjr W0403 00:04:53.916816 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:04:53.916862 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:04:53.916869 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:04:53.939350 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:04:54.923904 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0403 00:04:54.923940 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0403 00:04:54.923980 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0403 00:04:54.942747 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:04:55.934889 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0403 00:05:04.456158 3870 cni.go:403] [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default] I0403 00:05:04.530752 3870 cni.go:424] [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-7c687859fb-z2cst a108ce29037c1595d68ce37c221661bac49fbb6ee4d45dafb13c526b59ff32c5 network default NAD default], result "{}", err I0403 00:05:04.599473 3870 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c687859fb-z2cst of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:04.599512 3870 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7c687859fb-z2cst I0403 00:05:04.600852 3870 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7c687859fb-z2cst, ips: 10.128.0.7 I0403 00:05:04.600882 3870 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c687859fb-z2cst of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:04.962877 3870 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c687859fb-z2cst of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:04.962903 3870 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c687859fb-z2cst of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:04.968300 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c687859fb-z2cst I0403 00:05:04.968328 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c687859fb-z2cst I0403 00:05:07.989213 3870 cni.go:403] [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default] DEL starting CNI request [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default] I0403 00:05:08.079645 3870 cni.go:424] [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default] DEL finished CNI request [openshift-image-registry/image-pruner-29586240-mslpc b91b9655de35e08cf355d90a2a73b5a0be7a26aa7647e90c7f788c186aa37d31 network default NAD default], result "{}", err I0403 00:05:08.154569 3870 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29586240-mslpc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:08.154606 3870 pods.go:176] Deleting pod: openshift-image-registry/image-pruner-29586240-mslpc I0403 00:05:08.155884 3870 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-pruner-29586240-mslpc, ips: 10.128.0.6 I0403 00:05:08.155916 3870 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29586240-mslpc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0403 00:05:08.992254 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0403 00:05:08.992261 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0403 00:05:08.992456 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0403 00:05:09.007314 3870 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29586240-mslpc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:09.007339 3870 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29586240-mslpc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0403 00:05:10.004323 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0403 00:05:19.268551 3870 obj_retry.go:464] Detected object openshift-console/console-86d9fd7876-4vf2f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:19.269091 3870 obj_retry.go:464] Detected object openshift-console/console-86d9fd7876-4vf2f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:19.465513 3870 obj_retry.go:458] Detected object openshift-console/console-86d9fd7876-4vf2f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:19.465537 3870 obj_retry.go:458] Detected object openshift-console/console-86d9fd7876-4vf2f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:19.470743 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86d9fd7876-4vf2f I0403 00:05:19.470774 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86d9fd7876-4vf2f I0403 00:05:21.216913 3870 cni.go:403] [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default] DEL starting CNI request [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default] I0403 00:05:21.307293 3870 cni.go:424] [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default] DEL finished CNI request [openshift-ingress/router-default-76c85c67b7-g4kf5 d620160f287a7335af958e78a1339834731d16ffbaf7b5db4e34493e9c7f48bc network default NAD default], result "{}", err I0403 00:05:21.380429 3870 obj_retry.go:464] Detected object openshift-ingress/router-default-76c85c67b7-g4kf5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:21.380472 3870 pods.go:176] Deleting pod: openshift-ingress/router-default-76c85c67b7-g4kf5 I0403 00:05:21.381829 3870 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-76c85c67b7-g4kf5, ips: 10.128.0.16 I0403 00:05:21.381870 3870 obj_retry.go:464] Detected object openshift-ingress/router-default-76c85c67b7-g4kf5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:05:22.052707 3870 obj_retry.go:458] Detected object openshift-ingress/router-default-76c85c67b7-g4kf5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:22.052729 3870 obj_retry.go:458] Detected object openshift-ingress/router-default-76c85c67b7-g4kf5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:05:22.059999 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-76c85c67b7-g4kf5 I0403 00:05:22.060027 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-76c85c67b7-g4kf5 I0403 00:06:04.467392 3870 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0403 00:06:09.794110 3870 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-78b7847f64-tkcfs IP address from the namespace address-set, err: pod openshift-console/console-78b7847f64-tkcfs: no pod IPs found I0403 00:06:26.214302 3870 obj_retry.go:464] Detected object openshift-console/console-db7994b7c-qs6ts of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:06:26.214875 3870 obj_retry.go:464] Detected object openshift-console/console-db7994b7c-qs6ts of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:06:27.025639 3870 obj_retry.go:458] Detected object openshift-console/console-db7994b7c-qs6ts of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:06:27.025662 3870 obj_retry.go:458] Detected object openshift-console/console-db7994b7c-qs6ts of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:06:27.031256 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-db7994b7c-qs6ts I0403 00:06:27.031272 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-db7994b7c-qs6ts I0403 00:06:35.079993 3870 obj_retry.go:464] Detected object openshift-console/console-78b7847f64-8mlvw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:06:35.080576 3870 obj_retry.go:464] Detected object openshift-console/console-78b7847f64-8mlvw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:06:35.782007 3870 obj_retry.go:458] Detected object openshift-console/console-78b7847f64-8mlvw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:06:35.782031 3870 obj_retry.go:458] Detected object openshift-console/console-78b7847f64-8mlvw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:06:35.785898 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-78b7847f64-8mlvw I0403 00:06:35.785918 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-78b7847f64-8mlvw I0403 00:06:40.402080 3870 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default] I0403 00:06:40.482580 3870 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb 9f39427013697b48c4a25eddaafc9f05d91ab32a1199bdd7ea1f66e50f491a18 network default NAD default], result "{}", err I0403 00:06:40.524502 3870 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:06:40.524544 3870 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb I0403 00:06:40.525847 3870 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb, ips: 10.128.0.22 I0403 00:06:40.525881 3870 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:06:41.414052 3870 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:06:41.414102 3870 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586240-2xmxb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:07:02.256414 3870 cni.go:403] [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default] DEL starting CNI request [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default] I0403 00:07:02.344855 3870 cni.go:424] [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default] DEL finished CNI request [openshift-console/console-db7994b7c-6lh5g 166e96df0bf9f55c29bfe094cc005b835ca476478c77bf09f42b0229a12b148f network default NAD default], result "{}", err I0403 00:07:02.391082 3870 obj_retry.go:464] Detected object openshift-console/console-db7994b7c-6lh5g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:07:02.391121 3870 pods.go:176] Deleting pod: openshift-console/console-db7994b7c-6lh5g I0403 00:07:02.392430 3870 pods.go:236] Attempting to release IPs for pod: openshift-console/console-db7994b7c-6lh5g, ips: 10.128.0.30 I0403 00:07:02.392466 3870 obj_retry.go:464] Detected object openshift-console/console-db7994b7c-6lh5g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:07:02.783707 3870 obj_retry.go:458] Detected object openshift-console/console-db7994b7c-6lh5g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:07:02.783731 3870 obj_retry.go:458] Detected object openshift-console/console-db7994b7c-6lh5g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:07:02.790789 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-db7994b7c-6lh5g I0403 00:07:02.790816 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-db7994b7c-6lh5g I0403 00:08:29.527986 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=4 I0403 00:08:33.512181 3870 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0403 00:09:34.424678 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=120 I0403 00:09:46.413405 3870 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-c5x5s for pod on switch ip-10-0-2-108.ec2.internal I0403 00:09:46.413521 3870 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s I0403 00:09:46.437563 3870 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s] pod update took 24.06882ms I0403 00:09:46.437589 3870 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s] addLogicalPort annotation time took 24.09896ms I0403 00:09:46.438831 3870 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s] addLogicalPort took 25.443603ms, libovsdb time 873.301µs I0403 00:09:46.769790 3870 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s cee5781dd62f847c540e3d480c120f09b90dc3de8e0831a4b45435736624c606 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s cee5781dd62f847c540e3d480c120f09b90dc3de8e0831a4b45435736624c606 network default NAD default] I0403 00:09:46.780087 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-c5x5s, hostIfaceName: cee5781dd62f847, network: default, NAD default, SandboxID: "cee5781dd62f847c540e3d480c120f09b90dc3de8e0831a4b45435736624c606", PCI device ID: , UID: "7e26279c-95ce-42ed-ba48-57e315f49a1f", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0403 00:09:46.828013 3870 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s cee5781dd62f847c540e3d480c120f09b90dc3de8e0831a4b45435736624c606 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-c5x5s cee5781dd62f847c540e3d480c120f09b90dc3de8e0831a4b45435736624c606 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:81:7e:f5:d5:7f\",\"name\":\"cee5781dd62f847\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0f86280b-ccc2-48f6-b694-124f4c4523ee\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:09:50.207234 3870 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:10:00.179661 3870 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29586250-dh85d for pod on switch ip-10-0-2-108.ec2.internal I0403 00:10:00.179783 3870 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d I0403 00:10:00.192916 3870 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d] pod update took 13.15543ms I0403 00:10:00.192941 3870 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d] addLogicalPort annotation time took 13.183807ms I0403 00:10:00.194274 3870 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d] addLogicalPort took 14.631171ms, libovsdb time 975.459µs I0403 00:10:00.549990 3870 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default] I0403 00:10:00.559532 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29586250-dh85d, hostIfaceName: 71e0f6e14266433, network: default, NAD default, SandboxID: "71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff", PCI device ID: , UID: "2878974b-a261-474c-9246-3b939e42db72", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0403 00:10:00.608914 3870 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:ca:d1:35:7e:ff\",\"name\":\"71e0f6e14266433\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f66aebf2-2a52-4359-b8c1-559afda737bd\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:10:01.218373 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:10:01.251964 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0403 00:10:01.252020 3870 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-qb9kb IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-qb9kb: no pod IPs found I0403 00:10:01.521910 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0403 00:10:04.276397 3870 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default] I0403 00:10:04.352656 3870 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d 71e0f6e14266433bab912d24430430b814458f429ecef54cd33a06cb8ec2f3ff network default NAD default], result "{}", err I0403 00:10:04.394367 3870 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:10:04.394424 3870 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d I0403 00:10:04.395637 3870 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d, ips: 10.128.0.32 I0403 00:10:04.395666 3870 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:10:05.417217 3870 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:10:05.417237 3870 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586250-dh85d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:10:36.428248 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0403 00:10:38.551690 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0403 00:10:38.551697 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0403 00:11:07.427909 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 I0403 00:11:21.354402 3870 namespace.go:98] [tsf] adding namespace I0403 00:11:21.355268 3870 namespace.go:102] [tsf] adding namespace took 845.751µs I0403 00:11:21.372814 3870 namespace.go:142] [tsf] updating namespace I0403 00:11:21.383276 3870 namespace.go:142] [tsf] updating namespace I0403 00:11:25.209392 3870 namespace.go:98] [cert-manager-operator] adding namespace I0403 00:11:25.210356 3870 namespace.go:102] [cert-manager-operator] adding namespace took 936.421µs I0403 00:11:25.212747 3870 namespace.go:98] [tsf-tpa] adding namespace I0403 00:11:25.212807 3870 namespace.go:98] [tsf-tas] adding namespace I0403 00:11:25.213715 3870 namespace.go:102] [tsf-tpa] adding namespace took 946.82µs I0403 00:11:25.214347 3870 namespace.go:102] [tsf-tas] adding namespace took 1.526415ms I0403 00:11:25.214699 3870 namespace.go:98] [konflux-operator] adding namespace I0403 00:11:25.214736 3870 namespace.go:98] [tsf-keycloak] adding namespace I0403 00:11:25.214874 3870 namespace.go:98] [rhbk-operator] adding namespace I0403 00:11:25.215027 3870 namespace.go:98] [konflux-ui] adding namespace I0403 00:11:25.215391 3870 namespace.go:102] [konflux-operator] adding namespace took 676.689µs I0403 00:11:25.215887 3870 namespace.go:102] [tsf-keycloak] adding namespace took 1.092306ms I0403 00:11:25.215906 3870 namespace.go:98] [rhtpa-operator] adding namespace I0403 00:11:25.216397 3870 namespace.go:102] [rhtpa-operator] adding namespace took 483.247µs I0403 00:11:25.216921 3870 namespace.go:102] [rhbk-operator] adding namespace took 2.030284ms I0403 00:11:25.217488 3870 namespace.go:102] [konflux-ui] adding namespace took 2.441986ms I0403 00:11:25.230296 3870 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:11:25.242717 3870 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:11:25.251178 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:11:25.266031 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:11:25.269471 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:11:25.269505 3870 namespace.go:142] [konflux-operator] updating namespace I0403 00:11:25.271289 3870 namespace.go:142] [tsf-tas] updating namespace I0403 00:11:25.281610 3870 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:11:25.281703 3870 namespace.go:142] [rhbk-operator] updating namespace I0403 00:11:25.281754 3870 namespace.go:142] [konflux-ui] updating namespace I0403 00:11:25.281835 3870 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:11:25.292892 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:11:25.308472 3870 namespace.go:142] [konflux-operator] updating namespace I0403 00:11:25.319475 3870 namespace.go:142] [konflux-operator] updating namespace I0403 00:11:25.329254 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:11:25.344154 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:11:25.349287 3870 namespace.go:142] [rhbk-operator] updating namespace I0403 00:11:25.364701 3870 namespace.go:142] [rhbk-operator] updating namespace I0403 00:11:25.364791 3870 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:11:25.375810 3870 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:11:25.383679 3870 namespace.go:142] [konflux-ui] updating namespace I0403 00:11:25.397589 3870 namespace.go:142] [konflux-ui] updating namespace I0403 00:11:25.402889 3870 namespace.go:142] [tsf-tas] updating namespace I0403 00:11:25.413402 3870 namespace.go:142] [tsf-tas] updating namespace I0403 00:11:26.459831 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:11:26.460007 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:11:32.436173 3870 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:32.436773 3870 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:32.436799 3870 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:32.436807 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:11:32.436821 3870 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:32.436831 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:11:33.459408 3870 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:33.459433 3870 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:33.459452 3870 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:33.459464 3870 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebd8xvh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:34.085790 3870 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:34.086408 3870 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:34.086440 3870 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:34.086447 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:11:34.086459 3870 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:34.086463 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:11:35.111019 3870 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:35.111043 3870 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:35.111056 3870 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:35.111088 3870 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecebnv7c of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:36.423711 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=73 I0403 00:11:36.480461 3870 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:36.480979 3870 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:36.481011 3870 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:36.481021 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:11:36.481041 3870 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:36.481048 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:11:36.511770 3870 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0403 00:11:37.484128 3870 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:37.484653 3870 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:37.484682 3870 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:37.484690 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:11:37.484702 3870 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:37.484707 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:11:37.513207 3870 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:37.513231 3870 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:37.513247 3870 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:37.513261 3870 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c4dcb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:38.527571 3870 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:38.527598 3870 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:38.527616 3870 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:38.527628 3870 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1zrhjb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:39.520630 3870 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.521208 3870 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.521239 3870 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.521248 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:11:39.521260 3870 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.521264 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:11:39.554562 3870 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.555057 3870 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.555101 3870 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.555108 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:11:39.555121 3870 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:11:39.555125 3870 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:11:40.548365 3870 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.548390 3870 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.548407 3870 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.548424 3870 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm28jf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.586256 3870 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.586280 3870 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.586291 3870 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:40.586304 3870 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bthns of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:11:42.163689 3870 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k] creating logical port tsf-tpa_rhtpa-operator-controller-manager-576c85d8f-2r86k for pod on switch ip-10-0-2-108.ec2.internal I0403 00:11:42.163808 3870 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k I0403 00:11:42.180056 3870 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k] pod update took 16.269451ms I0403 00:11:42.180094 3870 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k] addLogicalPort annotation time took 16.313673ms I0403 00:11:42.181319 3870 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k] addLogicalPort took 17.65211ms, libovsdb time 854.885µs I0403 00:11:42.525335 3870 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k 3929a3fb7b2b9c0a4b5f1c04b197479604c63fd8c2646eb9969dbf9f95544a88 network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k 3929a3fb7b2b9c0a4b5f1c04b197479604c63fd8c2646eb9969dbf9f95544a88 network default NAD default] I0403 00:11:42.534395 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-576c85d8f-2r86k, hostIfaceName: 3929a3fb7b2b9c0, network: default, NAD default, SandboxID: "3929a3fb7b2b9c0a4b5f1c04b197479604c63fd8c2646eb9969dbf9f95544a88", PCI device ID: , UID: "d9c86a2b-9150-4f95-b951-e77b97b0e799", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0403 00:11:42.585196 3870 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k 3929a3fb7b2b9c0a4b5f1c04b197479604c63fd8c2646eb9969dbf9f95544a88 network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-2r86k 3929a3fb7b2b9c0a4b5f1c04b197479604c63fd8c2646eb9969dbf9f95544a88 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:94:2d:2d:84:21\",\"name\":\"3929a3fb7b2b9c0\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b51cccd0-cf5e-4f43-a33c-7b692c2075b7\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:11:43.435202 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=724 W0403 00:11:45.714578 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0403 00:11:48.546714 3870 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 W0403 00:11:50.479596 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0403 00:11:56.719990 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0403 00:12:12.212526 3870 namespace.go:98] [cert-manager] adding namespace I0403 00:12:12.213380 3870 namespace.go:102] [cert-manager] adding namespace took 833.106µs I0403 00:12:12.259279 3870 namespace.go:142] [cert-manager] updating namespace I0403 00:12:12.317545 3870 namespace.go:142] [cert-manager] updating namespace I0403 00:12:12.702720 3870 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-bjxgf] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-bjxgf for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:12.702832 3870 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-bjxgf I0403 00:12:12.803792 3870 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-bjxgf] pod update took 100.979949ms I0403 00:12:12.803823 3870 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-bjxgf] addLogicalPort annotation time took 101.019511ms I0403 00:12:12.805128 3870 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-bjxgf] addLogicalPort took 102.428162ms, libovsdb time 910.92µs I0403 00:12:12.993640 3870 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-bjxgf 9744e7d3411806311b60f0462f6861cc1208e3a6e533adf5b3292d02958aa60e network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-bjxgf 9744e7d3411806311b60f0462f6861cc1208e3a6e533adf5b3292d02958aa60e network default NAD default] I0403 00:12:13.002760 3870 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-bjxgf, hostIfaceName: 9744e7d34118063, network: default, NAD default, SandboxID: "9744e7d3411806311b60f0462f6861cc1208e3a6e533adf5b3292d02958aa60e", PCI device ID: , UID: "4ec34609-ccc9-4558-9402-b971ce8c2f8a", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0403 00:12:13.055115 3870 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-bjxgf 9744e7d3411806311b60f0462f6861cc1208e3a6e533adf5b3292d02958aa60e network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-bjxgf 9744e7d3411806311b60f0462f6861cc1208e3a6e533adf5b3292d02958aa60e network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:d5:cf:cc:f6:6b\",\"name\":\"9744e7d34118063\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e735f34-fcb9-4265-86ab-51c4994638b2\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:12:13.468487 3870 namespace.go:142] [openshift-operators] updating namespace I0403 00:12:13.509432 3870 namespace.go:142] [openshift-operators] updating namespace I0403 00:12:13.987502 3870 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-ctwrw for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:13.987615 3870 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw I0403 00:12:14.095494 3870 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw I0403 00:12:14.138165 3870 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw] pod update took 150.572768ms I0403 00:12:14.138193 3870 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw] addLogicalPort annotation time took 150.610614ms I0403 00:12:14.139453 3870 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw] addLogicalPort took 151.974206ms, libovsdb time 908.386µs I0403 00:12:14.371674 3870 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw 6b24f7a02ad102853705c27e0af788e1270c55f1f86ccf68c2b4f03e5572b4af network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw 6b24f7a02ad102853705c27e0af788e1270c55f1f86ccf68c2b4f03e5572b4af network default NAD default] I0403 00:12:14.381807 3870 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-ctwrw, hostIfaceName: 6b24f7a02ad1028, network: default, NAD default, SandboxID: "6b24f7a02ad102853705c27e0af788e1270c55f1f86ccf68c2b4f03e5572b4af", PCI device ID: , UID: "81aa3ec0-780f-4838-baff-d29d4dec3043", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0403 00:12:14.433808 3870 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw 6b24f7a02ad102853705c27e0af788e1270c55f1f86ccf68c2b4f03e5572b4af network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-ctwrw 6b24f7a02ad102853705c27e0af788e1270c55f1f86ccf68c2b4f03e5572b4af network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:c1:2a:d6:b6:03\",\"name\":\"6b24f7a02ad1028\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7cb32e8b-48dd-4970-b203-eb65d66c8cb7\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:12:17.427274 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=97 W0403 00:12:19.884390 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0403 00:12:19.884410 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0403 00:12:22.892366 3870 namespace.go:98] [trusted-artifact-signer] adding namespace I0403 00:12:22.893245 3870 namespace.go:102] [trusted-artifact-signer] adding namespace took 851.927µs I0403 00:12:22.913354 3870 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:12:22.928998 3870 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:12:23.052443 3870 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-7tcw4 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:23.052535 3870 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4 I0403 00:12:23.064625 3870 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4] pod update took 12.118663ms I0403 00:12:23.064648 3870 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4] addLogicalPort annotation time took 12.14786ms I0403 00:12:23.065860 3870 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4] addLogicalPort took 13.432506ms, libovsdb time 806.635µs I0403 00:12:23.410261 3870 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4 5b4a86a242026fca9da7efd4045f0e34a766b8e1b5b5b37381a98fdcac427298 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4 5b4a86a242026fca9da7efd4045f0e34a766b8e1b5b5b37381a98fdcac427298 network default NAD default] I0403 00:12:23.420077 3870 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-7tcw4, hostIfaceName: 5b4a86a242026fc, network: default, NAD default, SandboxID: "5b4a86a242026fca9da7efd4045f0e34a766b8e1b5b5b37381a98fdcac427298", PCI device ID: , UID: "739dd33f-82b6-4747-a3c6-36eb47f5059c", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0403 00:12:23.471885 3870 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4 5b4a86a242026fca9da7efd4045f0e34a766b8e1b5b5b37381a98fdcac427298 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-7tcw4 5b4a86a242026fca9da7efd4045f0e34a766b8e1b5b5b37381a98fdcac427298 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:02:dc:1a:02:25\",\"name\":\"5b4a86a242026fc\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32dfd98a-76b5-4e71-814e-48eaac5e3e42\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:12:23.633484 3870 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0403 00:12:24.894951 3870 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-pw6tl] creating logical port cert-manager_cert-manager-59ccb4bb6b-pw6tl for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:24.895084 3870 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-pw6tl I0403 00:12:24.918283 3870 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-pw6tl] pod update took 23.252773ms I0403 00:12:24.918306 3870 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-pw6tl] addLogicalPort annotation time took 23.292488ms I0403 00:12:24.919541 3870 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-pw6tl] addLogicalPort took 24.610552ms, libovsdb time 848.054µs I0403 00:12:25.254903 3870 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-pw6tl b5c4806aa8e3cc7383c703dc406c0694b38bc51627e5ef2c1099a2b1c1e4af16 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-pw6tl b5c4806aa8e3cc7383c703dc406c0694b38bc51627e5ef2c1099a2b1c1e4af16 network default NAD default] I0403 00:12:25.265485 3870 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-pw6tl, hostIfaceName: b5c4806aa8e3cc7, network: default, NAD default, SandboxID: "b5c4806aa8e3cc7383c703dc406c0694b38bc51627e5ef2c1099a2b1c1e4af16", PCI device ID: , UID: "db6ff2c6-7e87-400c-8073-d5e69f3a734a", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0403 00:12:25.324386 3870 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-pw6tl b5c4806aa8e3cc7383c703dc406c0694b38bc51627e5ef2c1099a2b1c1e4af16 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-pw6tl b5c4806aa8e3cc7383c703dc406c0694b38bc51627e5ef2c1099a2b1c1e4af16 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:d2:23:0c:c4:d3\",\"name\":\"b5c4806aa8e3cc7\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/957a4c17-40d8-4761-8598-d2e6f947c8a6\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:12:25.541184 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0403 00:12:25.731323 3870 namespace.go:142] [openshift-operators] updating namespace W0403 00:12:25.915424 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0403 00:12:25.915443 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0403 00:12:27.148036 3870 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-mpmx4 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:27.148223 3870 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4 I0403 00:12:27.193820 3870 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4] pod update took 45.646882ms I0403 00:12:27.193845 3870 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4] addLogicalPort annotation time took 45.678098ms I0403 00:12:27.195170 3870 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4] addLogicalPort took 47.154947ms, libovsdb time 933.764µs I0403 00:12:27.509589 3870 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4 797fc5652d2872968350c898e5a868e66b647cb5271e9f562a0098400e88f20b network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4 797fc5652d2872968350c898e5a868e66b647cb5271e9f562a0098400e88f20b network default NAD default] I0403 00:12:27.517340 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0403 00:12:27.520782 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-mpmx4, hostIfaceName: 797fc5652d28729, network: default, NAD default, SandboxID: "797fc5652d2872968350c898e5a868e66b647cb5271e9f562a0098400e88f20b", PCI device ID: , UID: "97cb279f-b230-4657-8de7-3b2a37e7e469", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0403 00:12:27.576126 3870 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4 797fc5652d2872968350c898e5a868e66b647cb5271e9f562a0098400e88f20b network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-mpmx4 797fc5652d2872968350c898e5a868e66b647cb5271e9f562a0098400e88f20b network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:8b:b5:dd:6d:42\",\"name\":\"797fc5652d28729\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/202f5da1-723f-4f0e-b9b8-57652e57c29e\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:12:36.321713 3870 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:12:36.322339 3870 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:12:36.343553 3870 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:12:36.343575 3870 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:12:36.348132 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0403 00:12:36.348165 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0403 00:12:41.661488 3870 namespace.go:98] [openshift-pipelines] adding namespace I0403 00:12:41.662377 3870 namespace.go:102] [openshift-pipelines] adding namespace took 862.497µs I0403 00:12:41.680698 3870 namespace.go:142] [openshift-pipelines] updating namespace I0403 00:12:42.393139 3870 namespace.go:142] [cert-manager] updating namespace I0403 00:12:42.590513 3870 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:12:42.992980 3870 namespace.go:142] [konflux-operator] updating namespace I0403 00:12:43.192755 3870 namespace.go:142] [konflux-ui] updating namespace I0403 00:12:43.397035 3870 namespace.go:142] [open-cluster-management-2peb42nlhreqhm7gegjie4h8sih13gfv] updating namespace I0403 00:12:43.798833 3870 namespace.go:142] [rhbk-operator] updating namespace I0403 00:12:44.001335 3870 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:12:44.206123 3870 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:12:44.407150 3870 namespace.go:142] [tsf] updating namespace I0403 00:12:44.608402 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:12:44.806177 3870 namespace.go:142] [tsf-tas] updating namespace I0403 00:12:45.009749 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:12:45.138382 3870 namespace.go:142] [cert-manager] updating namespace I0403 00:12:45.147678 3870 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:12:45.158621 3870 namespace.go:142] [default] updating namespace I0403 00:12:45.170202 3870 namespace.go:142] [konflux-operator] updating namespace I0403 00:12:45.181749 3870 namespace.go:142] [konflux-ui] updating namespace I0403 00:12:45.192035 3870 namespace.go:142] [open-cluster-management-2peb42nlhreqhm7gegjie4h8sih13gfv] updating namespace I0403 00:12:45.199396 3870 namespace.go:142] [openshift] updating namespace I0403 00:12:45.209673 3870 namespace.go:142] [rhbk-operator] updating namespace I0403 00:12:45.219046 3870 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:12:45.226330 3870 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:12:45.236123 3870 namespace.go:142] [tsf] updating namespace I0403 00:12:45.251653 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:12:45.271936 3870 namespace.go:142] [tsf-tas] updating namespace I0403 00:12:45.289548 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:12:45.388450 3870 namespace.go:142] [cert-manager] updating namespace I0403 00:12:45.491499 3870 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:12:45.593037 3870 namespace.go:142] [default] updating namespace I0403 00:12:45.691303 3870 namespace.go:142] [konflux-operator] updating namespace I0403 00:12:45.796647 3870 namespace.go:142] [konflux-ui] updating namespace I0403 00:12:45.891433 3870 namespace.go:142] [open-cluster-management-2peb42nlhreqhm7gegjie4h8sih13gfv] updating namespace I0403 00:12:45.993370 3870 namespace.go:142] [openshift] updating namespace I0403 00:12:46.092907 3870 namespace.go:142] [rhbk-operator] updating namespace I0403 00:12:46.191607 3870 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:12:46.288629 3870 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:12:46.396752 3870 namespace.go:142] [tsf] updating namespace I0403 00:12:46.491153 3870 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:12:46.590441 3870 namespace.go:142] [tsf-tas] updating namespace I0403 00:12:46.692186 3870 namespace.go:142] [tsf-tpa] updating namespace I0403 00:12:46.908947 3870 namespace.go:142] [openshift-pipelines] updating namespace W0403 00:12:49.252369 3870 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0403 00:12:49.274887 3870 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0403 00:12:54.422691 3870 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-7jqng for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:54.422792 3870 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng I0403 00:12:54.436551 3870 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng] pod update took 13.794216ms I0403 00:12:54.436574 3870 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng] addLogicalPort annotation time took 13.823552ms I0403 00:12:54.437938 3870 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng] addLogicalPort took 15.296381ms, libovsdb time 816.014µs I0403 00:12:54.637825 3870 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-k6d9s for pod on switch ip-10-0-2-108.ec2.internal I0403 00:12:54.637929 3870 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s I0403 00:12:54.651726 3870 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s] pod update took 13.823057ms I0403 00:12:54.651753 3870 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s] addLogicalPort annotation time took 13.854617ms I0403 00:12:54.653019 3870 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s] addLogicalPort took 15.212218ms, libovsdb time 846.419µs I0403 00:12:54.840781 3870 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng 75db7865f0549392f7ed248daaf3081b9de15804ee37c977881d66a734cfdcbb network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng 75db7865f0549392f7ed248daaf3081b9de15804ee37c977881d66a734cfdcbb network default NAD default] I0403 00:12:54.849983 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-7jqng, hostIfaceName: 75db7865f054939, network: default, NAD default, SandboxID: "75db7865f0549392f7ed248daaf3081b9de15804ee37c977881d66a734cfdcbb", PCI device ID: , UID: "bcb553d7-743f-4cfc-b3f5-3ed22786aaea", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0403 00:12:54.902962 3870 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng 75db7865f0549392f7ed248daaf3081b9de15804ee37c977881d66a734cfdcbb network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7jqng 75db7865f0549392f7ed248daaf3081b9de15804ee37c977881d66a734cfdcbb network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:26:ca:1d:5d:93\",\"name\":\"75db7865f054939\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10a59593-8726-4ea1-98d0-98de1bf0514f\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:12:55.045693 3870 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s d784b472901a8f2474e99c1f4380f7865f8501dea6761491e4163fd913dc9086 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s d784b472901a8f2474e99c1f4380f7865f8501dea6761491e4163fd913dc9086 network default NAD default] I0403 00:12:55.055802 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-k6d9s, hostIfaceName: d784b472901a8f2, network: default, NAD default, SandboxID: "d784b472901a8f2474e99c1f4380f7865f8501dea6761491e4163fd913dc9086", PCI device ID: , UID: "65a81ad2-e454-41af-a7b3-38e20d972f14", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0403 00:12:55.105580 3870 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s d784b472901a8f2474e99c1f4380f7865f8501dea6761491e4163fd913dc9086 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-k6d9s d784b472901a8f2474e99c1f4380f7865f8501dea6761491e4163fd913dc9086 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:39:d4:3a:0a:d2\",\"name\":\"d784b472901a8f2\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b2b46467-46fc-4ddb-90f6-1740f22e4653\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:12:55.399302 3870 egressfirewall.go:628] Syncing node "ip-10-0-1-110.ec2.internal" for egress firewall I0403 00:12:55.399375 3870 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-110.ec2.internal I0403 00:12:55.399402 3870 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-110.ec2.internal: 35.736µs W0403 00:12:57.806181 3870 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0403 00:12:57.806208 3870 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0403 00:12:57.806215 3870 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0403 00:12:59.829438 3870 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0403 00:12:59.829460 3870 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0403 00:12:59.829465 3870 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0403 00:12:59.848145 3870 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:12:59.848166 3870 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:12:59.848172 3870 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:12:59.848176 3870 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0403 00:13:10.534634 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0403 00:13:18.167725 3870 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-qdd6m for pod on switch ip-10-0-2-108.ec2.internal I0403 00:13:18.167823 3870 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m I0403 00:13:18.181157 3870 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m] pod update took 13.354732ms I0403 00:13:18.181181 3870 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m] addLogicalPort annotation time took 13.385018ms I0403 00:13:18.182653 3870 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m] addLogicalPort took 14.946904ms, libovsdb time 1.030281ms I0403 00:13:18.535080 3870 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m 5061a92e6a57276e6f297cc4eec66d034908487abcb759a57fc32ed1b61d5a99 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m 5061a92e6a57276e6f297cc4eec66d034908487abcb759a57fc32ed1b61d5a99 network default NAD default] I0403 00:13:18.544502 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-qdd6m, hostIfaceName: 5061a92e6a57276, network: default, NAD default, SandboxID: "5061a92e6a57276e6f297cc4eec66d034908487abcb759a57fc32ed1b61d5a99", PCI device ID: , UID: "5169c4ba-3816-4bf9-bf17-591f4acffd2e", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0403 00:13:18.597355 3870 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m 5061a92e6a57276e6f297cc4eec66d034908487abcb759a57fc32ed1b61d5a99 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-qdd6m 5061a92e6a57276e6f297cc4eec66d034908487abcb759a57fc32ed1b61d5a99 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:9b:0e:bd:1d:b1\",\"name\":\"5061a92e6a57276\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d1a3050c-ee90-41f5-9f5b-03cd76232b96\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:13:22.425469 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0403 00:13:27.574732 3870 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:13:27.575289 3870 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:13:27.624760 3870 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:13:27.624780 3870 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:13:27.630787 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0403 00:13:27.630810 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0403 00:13:28.755360 3870 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:13:28.755484 3870 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0403 00:13:28.772743 3870 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 17.287927ms I0403 00:13:28.772780 3870 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 17.335447ms I0403 00:13:28.774666 3870 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 19.326027ms, libovsdb time 1.274469ms I0403 00:13:29.129473 3870 cni.go:403] [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default] I0403 00:13:29.138552 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: f8bb86740dfe34a, network: default, NAD default, SandboxID: "f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf", PCI device ID: , UID: "00ab7798-cb89-465f-9935-4a1cd0e90659", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0403 00:13:29.189509 3870 cni.go:424] [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:e1:a4:f8:f1:05\",\"name\":\"f8bb86740dfe34a\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/abfc6182-b098-4621-94a5-2789b60e326f\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:13:33.531553 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 W0403 00:13:34.271845 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:13:34.271870 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:13:34.273409 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0403 00:13:40.429174 3870 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0403 00:13:53.410673 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0403 00:13:53.415333 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:13:53.415351 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:14:00.150457 3870 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:14:00.150484 3870 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:14:00.150491 3870 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0403 00:14:02.770190 3870 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:02.770748 3870 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:02.790295 3870 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:02.790318 3870 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:02.795945 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0403 00:14:02.795969 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0403 00:14:05.761407 3870 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-59b54f9f99-pfq5b] creating logical port tsf-tas_trillian-logsigner-59b54f9f99-pfq5b for pod on switch ip-10-0-2-108.ec2.internal I0403 00:14:05.761505 3870 kube.go:256] Updating pod tsf-tas/trillian-logsigner-59b54f9f99-pfq5b I0403 00:14:05.775656 3870 pod.go:62] [tsf-tas/trillian-logsigner-59b54f9f99-pfq5b] pod update took 14.169275ms I0403 00:14:05.775689 3870 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logsigner-59b54f9f99-pfq5b] addLogicalPort annotation time took 14.206752ms I0403 00:14:05.777152 3870 pods.go:271] [tsf-tas/trillian-logsigner-59b54f9f99-pfq5b] addLogicalPort took 15.765011ms, libovsdb time 903.03µs I0403 00:14:05.937955 3870 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-8ttzd-xdb88] creating logical port tsf-tas_rekor-createtree-job-8ttzd-xdb88 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:14:05.938089 3870 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-8ttzd-xdb88 I0403 00:14:05.950945 3870 pod.go:62] [tsf-tas/rekor-createtree-job-8ttzd-xdb88] pod update took 12.911866ms I0403 00:14:05.950975 3870 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-8ttzd-xdb88] addLogicalPort annotation time took 12.948842ms I0403 00:14:05.952657 3870 pods.go:271] [tsf-tas/rekor-createtree-job-8ttzd-xdb88] addLogicalPort took 14.723673ms, libovsdb time 1.078001ms I0403 00:14:06.299880 3870 cni.go:403] [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default] I0403 00:14:06.309217 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-8ttzd-xdb88, hostIfaceName: b9cbc0bb4897cbd, network: default, NAD default, SandboxID: "b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d", PCI device ID: , UID: "e2fbd066-f485-496f-9ff6-8a9904addca1", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0403 00:14:06.360240 3870 cni.go:424] [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:3a:31:43:d6:d6\",\"name\":\"b9cbc0bb4897cbd\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4eccba98-cc08-4bef-9491-a2aff1b2150c\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:14:06.725659 3870 cni.go:403] [tsf-tas/trillian-logsigner-59b54f9f99-pfq5b f70e68bd56707ecbeae8ab2d19221d83c607569a5b40c0800d1b5af2a6878260 network default NAD default] ADD starting CNI request [tsf-tas/trillian-logsigner-59b54f9f99-pfq5b f70e68bd56707ecbeae8ab2d19221d83c607569a5b40c0800d1b5af2a6878260 network default NAD default] I0403 00:14:06.736000 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-59b54f9f99-pfq5b, hostIfaceName: f70e68bd56707ec, network: default, NAD default, SandboxID: "f70e68bd56707ecbeae8ab2d19221d83c607569a5b40c0800d1b5af2a6878260", PCI device ID: , UID: "4e77f2ef-dd7c-4be2-a87a-5840134d428f", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0403 00:14:06.786275 3870 cni.go:424] [tsf-tas/trillian-logsigner-59b54f9f99-pfq5b f70e68bd56707ecbeae8ab2d19221d83c607569a5b40c0800d1b5af2a6878260 network default NAD default] ADD finished CNI request [tsf-tas/trillian-logsigner-59b54f9f99-pfq5b f70e68bd56707ecbeae8ab2d19221d83c607569a5b40c0800d1b5af2a6878260 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:22:48:23:1c:fb\",\"name\":\"f70e68bd56707ec\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/493340d7-60d3-4baa-8978-5676dac65dfb\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:14:12.505192 3870 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0403 00:14:19.290835 3870 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0403 00:14:22.312012 3870 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0403 00:14:23.461962 3870 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-74m9v-hddlr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:23.462583 3870 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-74m9v-hddlr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:24.353424 3870 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-74m9v-hddlr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:24.353448 3870 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-74m9v-hddlr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:28.544639 3870 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-f89k4 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:14:28.544731 3870 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4 I0403 00:14:28.557976 3870 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4] pod update took 13.264598ms I0403 00:14:28.558008 3870 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4] addLogicalPort annotation time took 13.301082ms I0403 00:14:28.559646 3870 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4] addLogicalPort took 15.025896ms, libovsdb time 953.891µs I0403 00:14:28.908525 3870 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4 34acb651efd634e32d23f61e5e7edc1c41ad4d5924e6e2d74ce8a0cd83b04929 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4 34acb651efd634e32d23f61e5e7edc1c41ad4d5924e6e2d74ce8a0cd83b04929 network default NAD default] I0403 00:14:28.936638 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-f89k4, hostIfaceName: 34acb651efd634e, network: default, NAD default, SandboxID: "34acb651efd634e32d23f61e5e7edc1c41ad4d5924e6e2d74ce8a0cd83b04929", PCI device ID: , UID: "e5341c3a-9d61-467a-a06c-f6f12d2a1090", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0403 00:14:28.941896 3870 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-s785n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:28.942505 3870 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-s785n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:28.987961 3870 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4 34acb651efd634e32d23f61e5e7edc1c41ad4d5924e6e2d74ce8a0cd83b04929 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-f89k4 34acb651efd634e32d23f61e5e7edc1c41ad4d5924e6e2d74ce8a0cd83b04929 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:cf:49:b9:af:77\",\"name\":\"34acb651efd634e\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3f82e1a9-fa2f-4b22-b237-a6802bce531a\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:14:29.967455 3870 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-s785n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:29.967484 3870 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-s785n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:30.207364 3870 cni.go:403] [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default] I0403 00:14:30.285238 3870 cni.go:424] [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 f8bb86740dfe34a8b8f2afcafd4413ddb6cfdb2c8f4d6496786513a8e2f6b4bf network default NAD default], result "{}", err I0403 00:14:30.557023 3870 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:30.557080 3870 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0403 00:14:30.558565 3870 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.42 I0403 00:14:30.558599 3870 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:30.869811 3870 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:30.869887 3870 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:30.875635 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0403 00:14:30.875668 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0403 00:14:30.908872 3870 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:14:30.908958 3870 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0403 00:14:30.922873 3870 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 13.933211ms I0403 00:14:30.922903 3870 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 13.968965ms I0403 00:14:30.924563 3870 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 15.705182ms, libovsdb time 1.077633ms W0403 00:14:31.369754 3870 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0403 00:14:31.369781 3870 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0403 00:14:31.393579 3870 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0403 00:14:31.569603 3870 cni.go:403] [tsf-keycloak/keycloak-0 f8cde894520fa64bf1d8196e26e903799c0bae1f4e1417b8469ed8f1c2ca8fa7 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 f8cde894520fa64bf1d8196e26e903799c0bae1f4e1417b8469ed8f1c2ca8fa7 network default NAD default] I0403 00:14:31.579448 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: f8cde894520fa64, network: default, NAD default, SandboxID: "f8cde894520fa64bf1d8196e26e903799c0bae1f4e1417b8469ed8f1c2ca8fa7", PCI device ID: , UID: "02b20103-8af3-4bbf-aea3-0ee820be0ba9", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0403 00:14:31.632050 3870 cni.go:424] [tsf-keycloak/keycloak-0 f8cde894520fa64bf1d8196e26e903799c0bae1f4e1417b8469ed8f1c2ca8fa7 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 f8cde894520fa64bf1d8196e26e903799c0bae1f4e1417b8469ed8f1c2ca8fa7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:07:03:cc:17:8e\",\"name\":\"f8cde894520fa64\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24a50b9a-0147-480e-8dd0-4109906cf443\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:14:37.625960 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0403 00:14:37.625961 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:14:37.626018 3870 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0403 00:14:51.427940 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=102 W0403 00:14:51.659398 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner W0403 00:14:51.901368 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:14:51.901399 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:14:51.901513 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0403 00:14:53.047670 3870 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:53.048282 3870 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:14:53.068087 3870 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:53.068106 3870 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:14:53.073905 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0403 00:14:53.073929 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0403 00:14:55.915999 3870 namespace.go:142] [openshift-pipelines] updating namespace I0403 00:15:03.119999 3870 obj_retry.go:464] Detected object tsf/patch-tekton-config-k9vzh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:03.120583 3870 obj_retry.go:464] Detected object tsf/patch-tekton-config-k9vzh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:04.012169 3870 obj_retry.go:458] Detected object tsf/patch-tekton-config-k9vzh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:04.012193 3870 obj_retry.go:458] Detected object tsf/patch-tekton-config-k9vzh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:08.515704 3870 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0403 00:15:10.160216 3870 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-gjprb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:10.160803 3870 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-gjprb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:10.754439 3870 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-lghsb-rdx9j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:10.755025 3870 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-lghsb-rdx9j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:10.811213 3870 cni.go:403] [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default] I0403 00:15:10.898129 3870 cni.go:424] [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-8ttzd-xdb88 b9cbc0bb4897cbd8c47955c8727dce5236f32a637ae975422f7eddcffddb449d network default NAD default], result "{}", err I0403 00:15:10.947509 3870 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-8ttzd-xdb88 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:10.947549 3870 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-8ttzd-xdb88 I0403 00:15:10.949012 3870 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-8ttzd-xdb88, ips: 10.128.0.44 I0403 00:15:10.949041 3870 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-8ttzd-xdb88 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:11.182009 3870 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-gjprb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.182032 3870 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-gjprb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.227664 3870 obj_retry.go:458] Detected object tsf/patch-tekton-config-k9vzh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.227682 3870 obj_retry.go:458] Detected object tsf/patch-tekton-config-k9vzh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.231958 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-k9vzh I0403 00:15:11.231985 3870 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-k9vzh I0403 00:15:11.622794 3870 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-lghsb-rdx9j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.622824 3870 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-lghsb-rdx9j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.826840 3870 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-8ttzd-xdb88 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:11.826864 3870 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-8ttzd-xdb88 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:17.367104 3870 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-d95dd559c-8qqdr] creating logical port tsf-tas_rekor-search-ui-d95dd559c-8qqdr for pod on switch ip-10-0-2-108.ec2.internal I0403 00:15:17.367222 3870 kube.go:256] Updating pod tsf-tas/rekor-search-ui-d95dd559c-8qqdr I0403 00:15:17.378875 3870 pod.go:62] [tsf-tas/rekor-search-ui-d95dd559c-8qqdr] pod update took 11.684575ms I0403 00:15:17.378900 3870 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-d95dd559c-8qqdr] addLogicalPort annotation time took 11.719105ms I0403 00:15:17.380232 3870 pods.go:271] [tsf-tas/rekor-search-ui-d95dd559c-8qqdr] addLogicalPort took 13.147697ms, libovsdb time 898.728µs I0403 00:15:17.740779 3870 cni.go:403] [tsf-tas/rekor-search-ui-d95dd559c-8qqdr 3c5beb3683fe22fc9bed5049841440108b17642f4b1aa8cb36d16bb12353bc2b network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-d95dd559c-8qqdr 3c5beb3683fe22fc9bed5049841440108b17642f4b1aa8cb36d16bb12353bc2b network default NAD default] I0403 00:15:17.750210 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-d95dd559c-8qqdr, hostIfaceName: 3c5beb3683fe22f, network: default, NAD default, SandboxID: "3c5beb3683fe22fc9bed5049841440108b17642f4b1aa8cb36d16bb12353bc2b", PCI device ID: , UID: "5f76456a-fd07-4e8f-be15-cce42229ea5e", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0403 00:15:17.806610 3870 cni.go:424] [tsf-tas/rekor-search-ui-d95dd559c-8qqdr 3c5beb3683fe22fc9bed5049841440108b17642f4b1aa8cb36d16bb12353bc2b network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-d95dd559c-8qqdr 3c5beb3683fe22fc9bed5049841440108b17642f4b1aa8cb36d16bb12353bc2b network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:03:38:4e:0e:3c\",\"name\":\"3c5beb3683fe22f\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d13087b-e138-4a4e-aeb5-ebe6f6784afd\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:15:19.077051 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0403 00:15:19.664131 3870 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0403 00:15:19.710642 3870 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-f8b8d49cc-gv958] creating logical port tsf-tas_rekor-server-f8b8d49cc-gv958 for pod on switch ip-10-0-2-108.ec2.internal I0403 00:15:19.710835 3870 kube.go:256] Updating pod tsf-tas/rekor-server-f8b8d49cc-gv958 I0403 00:15:19.723233 3870 pod.go:62] [tsf-tas/rekor-server-f8b8d49cc-gv958] pod update took 12.434465ms I0403 00:15:19.723262 3870 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-f8b8d49cc-gv958] addLogicalPort annotation time took 12.469147ms I0403 00:15:19.724913 3870 pods.go:271] [tsf-tas/rekor-server-f8b8d49cc-gv958] addLogicalPort took 14.290186ms, libovsdb time 1.091543ms W0403 00:15:20.864116 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0403 00:15:29.382008 3870 cni.go:403] [tsf-tas/rekor-server-f8b8d49cc-gv958 1b3b3487f851527f912a7f17c3bd25bc1e9fbc0e557f21234628dec08a28addd network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-f8b8d49cc-gv958 1b3b3487f851527f912a7f17c3bd25bc1e9fbc0e557f21234628dec08a28addd network default NAD default] I0403 00:15:29.391626 3870 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-f8b8d49cc-gv958, hostIfaceName: 1b3b3487f851527, network: default, NAD default, SandboxID: "1b3b3487f851527f912a7f17c3bd25bc1e9fbc0e557f21234628dec08a28addd", PCI device ID: , UID: "48e6a0c2-7aca-49a2-9edf-81e128be6d07", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0403 00:15:29.444316 3870 cni.go:424] [tsf-tas/rekor-server-f8b8d49cc-gv958 1b3b3487f851527f912a7f17c3bd25bc1e9fbc0e557f21234628dec08a28addd network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-f8b8d49cc-gv958 1b3b3487f851527f912a7f17c3bd25bc1e9fbc0e557f21234628dec08a28addd network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:37:d8:20:f7:34\",\"name\":\"1b3b3487f851527\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a30b70c-a9ed-4876-9feb-3c447a6df657\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0403 00:15:31.921934 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0403 00:15:32.915881 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0403 00:15:40.555479 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 W0403 00:15:42.933751 3870 util.go:959] Failed to build node endpoints for node ip-10-0-2-108.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0403 00:15:57.399501 3870 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-k95vg-dmj7r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:57.400073 3870 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-k95vg-dmj7r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:15:58.289464 3870 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-k95vg-dmj7r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:15:58.289490 3870 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-k95vg-dmj7r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0403 00:16:04.311161 3870 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0403 00:17:25.431768 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=98 I0403 00:17:32.545726 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0403 00:17:55.466550 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=55 I0403 00:17:56.524828 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0403 00:18:58.443980 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=411 I0403 00:19:12.554877 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0403 00:19:24.515204 3870 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0403 00:19:27.523227 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0403 00:19:39.538240 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0403 00:20:00.194271 3870 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29586260-nqsfb for pod on switch ip-10-0-2-108.ec2.internal I0403 00:20:00.194389 3870 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb I0403 00:20:00.217657 3870 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb] pod update took 23.290267ms I0403 00:20:00.217682 3870 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb] addLogicalPort annotation time took 23.32089ms I0403 00:20:00.219139 3870 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb] addLogicalPort took 24.87993ms, libovsdb time 988.303µs I0403 00:20:00.560571 3870 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default] I0403 00:20:00.569601 3870 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29586260-nqsfb, hostIfaceName: 3571fb44d47af45, network: default, NAD default, SandboxID: "3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01", PCI device ID: , UID: "97b993b4-bfa2-4b6d-ad11-8f14edd2c437", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0403 00:20:00.622352 3870 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:45:a8:61:6b:bd\",\"name\":\"3571fb44d47af45\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/82e8b264-82ff-4281-87e2-f3bfbce655a3\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0403 00:20:04.348229 3870 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default] I0403 00:20:04.434373 3870 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb 3571fb44d47af457883abfe7ebd031cf9d65e968fd38968974874641bf57ad01 network default NAD default], result "{}", err I0403 00:20:04.482717 3870 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:20:04.482756 3870 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb I0403 00:20:04.484196 3870 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb, ips: 10.128.0.49 I0403 00:20:04.484225 3870 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:20:05.360134 3870 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:20:05.360165 3870 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586260-nqsfb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:20:17.431890 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0403 00:21:14.427127 3870 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0403 00:21:15.001277 3870 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:21:15.001842 3870 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:21:17.558324 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0403 00:21:20.534645 3870 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0403 00:21:43.550461 3870 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11