+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-167.ec2.internal ++ [[ -n ip-10-0-2-167.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-167.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0521 19:38:13.622159250 - disable conntrack on geneve port' I0521 19:38:13.622159250 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0521 19:38:13.640022493 - starting ovnkube-node + echo 'I0521 19:38:13.640022493 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-167.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-167.ec2.internal --init-node ip-10-0-2-167.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-167.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0521 19:38:13.680381 3643 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0521 19:38:13.680443 3643 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-a83b8e6563.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0521 19:38:13.681836 3643 kube.go:431] Waiting for certificate I0521 19:38:13.681897 3643 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0521 19:38:13.681978 3643 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0521 19:38:13.682837 3643 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0521 19:38:13.720438 3643 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:38:13.720457 3643 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:38:13.726166 3643 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:38:13.726259 3643 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-xq8gn" I0521 19:38:13.731870 3643 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-xq8gn" I0521 19:38:13.731939 3643 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:38:14.682864 3643 kube.go:438] Certificate found I0521 19:38:14.683521 3643 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0521 19:38:14.684380 3643 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0521 19:38:14.734426 3643 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-22 19:33:13 +0000 UTC" deadline="2026-05-22 16:21:50.884633624 +0000 UTC" I0521 19:38:14.734462 3643 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h43m36.150181793s" I0521 19:38:14.751767 3643 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0521 19:38:14.761967 3643 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0521 19:38:14.762027 3643 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0521 19:38:14.762054 3643 server.go:101] MetricServer registers OVS metrics I0521 19:38:14.762090 3643 factory.go:561] Starting watch factory I0521 19:38:14.762217 3643 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.762234 3643 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.762247 3643 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.762266 3643 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.762400 3643 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.762427 3643 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.762560 3643 server.go:105] MetricServer registers OVN DB metrics I0521 19:38:14.762888 3643 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.763024 3643 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.763406 3643 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.763419 3643 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.763427 3643 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.763437 3643 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.767540 3643 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0521 19:38:14.767558 3643 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0521 19:38:14.768972 3643 controller_manager.go:370] Starting the ovnkube controller I0521 19:38:14.768982 3643 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-167.ec2.internal I0521 19:38:14.769022 3643 controller_manager.go:395] NBDB zone sync took: 33.833µs I0521 19:38:14.769030 3643 factory.go:561] Starting watch factory I0521 19:38:14.780841 3643 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0521 19:38:14.780859 3643 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0521 19:38:14.784394 3643 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0521 19:38:14.784425 3643 server.go:109] MetricServer registers OVN Controller metrics I0521 19:38:14.785602 3643 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.785732 3643 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.786448 3643 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.787959 3643 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.790870 3643 server.go:113] MetricServer registers OVN Northd metrics I0521 19:38:14.792157 3643 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.795222 3643 metrics.go:519] OVN Metrics Server starts to run ... I0521 19:38:14.796381 3643 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:38:14.868038 3643 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0521 19:38:14.868061 3643 factory.go:1977] *v1.Namespace informer cache synced successfully I0521 19:38:14.868066 3643 factory.go:1977] *v1.Node informer cache synced successfully I0521 19:38:14.868069 3643 factory.go:1977] *v1.Service informer cache synced successfully I0521 19:38:14.868073 3643 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0521 19:38:14.868077 3643 factory.go:1977] *v1.Pod informer cache synced successfully I0521 19:38:14.868214 3643 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:38:14.868242 3643 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:38:14.868351 3643 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:38:14.868382 3643 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:38:14.872298 3643 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:38:14.872315 3643 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:38:14.874146 3643 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0521 19:38:14.874157 3643 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0521 19:38:14.874161 3643 factory.go:1977] *v1.Node informer cache synced successfully I0521 19:38:14.874173 3643 factory.go:1977] *v1.Service informer cache synced successfully I0521 19:38:14.874176 3643 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0521 19:38:14.874179 3643 factory.go:1977] *v1.Pod informer cache synced successfully I0521 19:38:14.874183 3643 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0521 19:38:14.874186 3643 factory.go:1977] *v1.Namespace informer cache synced successfully I0521 19:38:14.874224 3643 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.874238 3643 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.878472 3643 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.879392 3643 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0521 19:38:14.879406 3643 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0521 19:38:14.879398 3643 factory.go:1977] *v1.EgressIP informer cache synced successfully I0521 19:38:14.879469 3643 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.879482 3643 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.882974 3643 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.884916 3643 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0521 19:38:14.884922 3643 factory.go:1977] *v1.EgressIP informer cache synced successfully I0521 19:38:14.884991 3643 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.885009 3643 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.888363 3643 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.890395 3643 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0521 19:38:14.890417 3643 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0521 19:38:14.890470 3643 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.890487 3643 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.893636 3643 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.895639 3643 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0521 19:38:14.895715 3643 factory.go:1977] *v1.EgressService informer cache synced successfully I0521 19:38:14.895784 3643 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.895794 3643 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.899033 3643 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.900963 3643 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0521 19:38:14.900975 3643 factory.go:1977] *v1.EgressService informer cache synced successfully I0521 19:38:14.901246 3643 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0521 19:38:14.901263 3643 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0521 19:38:14.905146 3643 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0521 19:38:14.906106 3643 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0521 19:38:14.906171 3643 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0521 19:38:14.906231 3643 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.906248 3643 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.906294 3643 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.906311 3643 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.909618 3643 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.909830 3643 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:38:14.911880 3643 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0521 19:38:14.911892 3643 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0521 19:38:14.911899 3643 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0521 19:38:14.911918 3643 factory.go:680] Watch Factory start up complete, took: 149.830972ms I0521 19:38:14.912024 3643 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0521 19:38:14.917174 3643 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0521 19:38:14.917188 3643 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0521 19:38:14.917195 3643 factory.go:680] Watch Factory start up complete, took: 148.163ms I0521 19:38:14.917203 3643 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-167.ec2.internal" zone I0521 19:38:14.947315 3643 default_node_network_controller.go:710] Initializing the default node network controller I0521 19:38:15.006974 3643 udn_isolation.go:101] Starting UDN host isolation manager I0521 19:38:15.014560 3643 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0521 19:38:15.051525 3643 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0521 19:38:15.051616 3643 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0521 19:38:15.051637 3643 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0521 19:38:15.084235 3643 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0521 19:38:15.084300 3643 default_node_network_controller.go:809] Node ip-10-0-2-167.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0521 19:38:15.089921 3643 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.167"] k8s.ovn.org/zone-name:ip-10-0-2-167.ec2.internal] on node ip-10-0-2-167.ec2.internal I0521 19:38:15.112676 3643 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0521 19:38:15.118805 3643 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0521 19:38:15.118924 3643 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0521 19:38:15.119083 3643 gateway_init.go:263] Preparing Gateway I0521 19:38:15.119094 3643 gateway_shared_intf.go:1687] Creating new gateway I0521 19:38:15.168138 3643 controller_manager.go:429] Waiting for node in zone sync took: 250.922633ms I0521 19:38:15.174222 3643 controller_manager.go:611] Switching to transit router for layer2 networks I0521 19:38:15.174235 3643 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-167.ec2.internal I0521 19:38:15.190895 3643 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0521 19:38:15.198137 3643 controller_manager.go:290] SCTP support detected in OVN I0521 19:38:15.198769 3643 services_controller.go:69] Creating services controller for network=default I0521 19:38:15.198861 3643 udn_enabled_svc.go:80] Waiting for service informer to sync I0521 19:38:15.198891 3643 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0521 19:38:15.198944 3643 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0521 19:38:15.198959 3643 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0521 19:38:15.198968 3643 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0521 19:38:15.199143 3643 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0521 19:38:15.199195 3643 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0521 19:38:15.199262 3643 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 79.159µs I0521 19:38:15.199297 3643 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 92.125µs I0521 19:38:15.199311 3643 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 110.084µs I0521 19:38:15.199322 3643 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0521 19:38:15.199331 3643 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0521 19:38:15.199343 3643 default_network_controller.go:325] Starting the default network controller I0521 19:38:15.199394 3643 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.452µs I0521 19:38:15.199427 3643 udn_enabled_svc.go:95] Performing full resync I0521 19:38:15.199801 3643 udn_enabled_svc.go:99] Waiting for handler to sync I0521 19:38:15.199813 3643 udn_enabled_svc.go:104] Starting worker I0521 19:38:15.204159 3643 iptables.go:108] Creating table: filter chain: FORWARD I0521 19:38:15.209885 3643 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0521 19:38:15.209958 3643 repair.go:33] Syncing exgw routes took 60.861µs I0521 19:38:15.209987 3643 default_network_controller.go:414] Starting all the Watchers... I0521 19:38:15.210117 3643 namespace.go:98] [openshift-config] adding namespace I0521 19:38:15.210122 3643 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0521 19:38:15.210129 3643 namespace.go:98] [openshift-backplane-mobb] adding namespace I0521 19:38:15.210138 3643 namespace.go:98] [openshift-apiserver] adding namespace I0521 19:38:15.210143 3643 namespace.go:98] [openshift-dns-operator] adding namespace I0521 19:38:15.210146 3643 namespace.go:98] [openshift-network-node-identity] adding namespace I0521 19:38:15.210152 3643 namespace.go:98] [openshift-apiserver-operator] adding namespace I0521 19:38:15.210157 3643 namespace.go:98] [openshift-console] adding namespace I0521 19:38:15.210163 3643 namespace.go:98] [openshift-authentication-operator] adding namespace I0521 19:38:15.210165 3643 namespace.go:98] [openshift-image-registry] adding namespace I0521 19:38:15.210169 3643 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0521 19:38:15.210169 3643 namespace.go:98] [openshift-kube-scheduler] adding namespace I0521 19:38:15.210175 3643 namespace.go:98] [openshift-ingress] adding namespace I0521 19:38:15.210176 3643 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0521 19:38:15.210117 3643 namespace.go:98] [kube-system] adding namespace I0521 19:38:15.210794 3643 iptables.go:108] Creating table: filter chain: OUTPUT I0521 19:38:15.210831 3643 namespace.go:102] [openshift-config] adding namespace took 701.03µs I0521 19:38:15.210851 3643 namespace.go:98] [openshift-package-operator] adding namespace I0521 19:38:15.211392 3643 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 1.258605ms I0521 19:38:15.211407 3643 namespace.go:98] [openshift-ingress-canary] adding namespace I0521 19:38:15.211848 3643 namespace.go:102] [openshift-ingress-canary] adding namespace took 433.527µs I0521 19:38:15.211865 3643 namespace.go:98] [openshift-node] adding namespace I0521 19:38:15.212475 3643 namespace.go:102] [openshift-backplane-mobb] adding namespace took 2.335209ms I0521 19:38:15.212495 3643 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0521 19:38:15.213008 3643 namespace.go:102] [openshift-apiserver] adding namespace took 2.857182ms I0521 19:38:15.213027 3643 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0521 19:38:15.213577 3643 namespace.go:102] [openshift-dns-operator] adding namespace took 3.423874ms I0521 19:38:15.213599 3643 namespace.go:98] [openshift-logging] adding namespace I0521 19:38:15.214151 3643 namespace.go:102] [openshift-network-node-identity] adding namespace took 3.99842ms I0521 19:38:15.214166 3643 namespace.go:98] [openshift-backplane-cse] adding namespace I0521 19:38:15.214711 3643 namespace.go:102] [openshift-apiserver-operator] adding namespace took 4.548823ms I0521 19:38:15.214727 3643 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0521 19:38:15.215655 3643 namespace.go:102] [openshift-console] adding namespace took 5.488816ms I0521 19:38:15.215674 3643 namespace.go:98] [openshift-network-operator] adding namespace I0521 19:38:15.217161 3643 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.991044ms I0521 19:38:15.217176 3643 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0521 19:38:15.217286 3643 gateway_shared_intf.go:1816] Gateway Creation Complete I0521 19:38:15.217300 3643 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-167.ec2.internal","mac-address":"0e:a3:e7:4e:db:71","ip-addresses":["10.0.2.167/24"],"ip-address":"10.0.2.167/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:fb42eae9-0346-4884-94e5-86d8ea9cc637 k8s.ovn.org/node-encap-ips:["10.0.2.167"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.167/24"} k8s.ovn.org/zone-name:ip-10-0-2-167.ec2.internal] on node ip-10-0-2-167.ec2.internal I0521 19:38:15.217800 3643 namespace.go:102] [openshift-image-registry] adding namespace took 7.621348ms I0521 19:38:15.217822 3643 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0521 19:38:15.218362 3643 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 8.183762ms I0521 19:38:15.218379 3643 namespace.go:98] [openshift-backplane-cee] adding namespace I0521 19:38:15.218839 3643 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.657296ms I0521 19:38:15.218858 3643 namespace.go:98] [openshift-marketplace] adding namespace I0521 19:38:15.219358 3643 namespace.go:102] [openshift-ingress] adding namespace took 9.176827ms I0521 19:38:15.219371 3643 namespace.go:98] [openshift-machine-config-operator] adding namespace W0521 19:38:15.219823 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-88jtx: pod kube-system/global-pull-secret-syncer-88jtx: no pod IPs found I0521 19:38:15.219852 3643 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 9.665179ms I0521 19:38:15.219876 3643 namespace.go:98] [openshift-dns] adding namespace I0521 19:38:15.220494 3643 namespace.go:102] [kube-system] adding namespace took 10.256741ms I0521 19:38:15.220508 3643 namespace.go:98] [openshift-multus] adding namespace I0521 19:38:15.220933 3643 namespace.go:102] [openshift-package-operator] adding namespace took 10.072294ms I0521 19:38:15.220949 3643 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0521 19:38:15.221362 3643 namespace.go:102] [openshift-node] adding namespace took 9.489353ms I0521 19:38:15.221379 3643 namespace.go:98] [open-cluster-management-2qegn84g18ids9l2o4klqb2qr2p2km0b] adding namespace I0521 19:38:15.221821 3643 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 9.318574ms I0521 19:38:15.221837 3643 namespace.go:98] [openshift-insights] adding namespace I0521 19:38:15.222243 3643 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 9.20998ms I0521 19:38:15.222258 3643 namespace.go:98] [dedicated-admin] adding namespace I0521 19:38:15.222634 3643 namespace.go:102] [openshift-logging] adding namespace took 9.029181ms I0521 19:38:15.222649 3643 namespace.go:98] [openshift-backplane] adding namespace I0521 19:38:15.223080 3643 namespace.go:102] [openshift-backplane-cse] adding namespace took 8.908415ms I0521 19:38:15.223099 3643 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0521 19:38:15.223481 3643 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.745543ms I0521 19:38:15.223497 3643 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0521 19:38:15.223883 3643 namespace.go:102] [openshift-network-operator] adding namespace took 8.203175ms I0521 19:38:15.223913 3643 namespace.go:98] [openshift-config-operator] adding namespace I0521 19:38:15.224306 3643 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.124868ms I0521 19:38:15.224322 3643 namespace.go:98] [default] adding namespace I0521 19:38:15.224710 3643 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.874118ms I0521 19:38:15.224727 3643 namespace.go:98] [kube-public] adding namespace I0521 19:38:15.225179 3643 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.792948ms I0521 19:38:15.225193 3643 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0521 19:38:15.225629 3643 namespace.go:102] [openshift-marketplace] adding namespace took 6.764916ms I0521 19:38:15.225641 3643 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0521 19:38:15.226117 3643 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.73937ms I0521 19:38:15.226131 3643 namespace.go:98] [openshift-authentication] adding namespace I0521 19:38:15.226559 3643 namespace.go:102] [openshift-dns] adding namespace took 6.678974ms W0521 19:38:15.226567 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-sc58z: pod openshift-multus/network-metrics-daemon-sc58z: no pod IPs found I0521 19:38:15.226570 3643 namespace.go:98] [openshift-backplane-ai-agent] adding namespace I0521 19:38:15.227321 3643 namespace.go:102] [openshift-multus] adding namespace took 6.806382ms I0521 19:38:15.227338 3643 namespace.go:98] [openshift-console-operator] adding namespace I0521 19:38:15.227834 3643 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.878348ms I0521 19:38:15.227847 3643 namespace.go:98] [openshift-backplane-srep] adding namespace I0521 19:38:15.228258 3643 namespace.go:102] [open-cluster-management-2qegn84g18ids9l2o4klqb2qr2p2km0b] adding namespace took 6.874487ms I0521 19:38:15.228270 3643 namespace.go:98] [kube-node-lease] adding namespace I0521 19:38:15.228769 3643 namespace.go:102] [openshift-insights] adding namespace took 6.926119ms I0521 19:38:15.228784 3643 namespace.go:98] [openshift-network-diagnostics] adding namespace I0521 19:38:15.229308 3643 namespace.go:102] [dedicated-admin] adding namespace took 7.043618ms I0521 19:38:15.229321 3643 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0521 19:38:15.229860 3643 namespace.go:102] [openshift-backplane] adding namespace took 7.202718ms I0521 19:38:15.229876 3643 namespace.go:98] [openshift] adding namespace I0521 19:38:15.230436 3643 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.331745ms I0521 19:38:15.230447 3643 namespace.go:98] [openshift-controller-manager] adding namespace I0521 19:38:15.230854 3643 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.351889ms I0521 19:38:15.230865 3643 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0521 19:38:15.231383 3643 namespace.go:102] [openshift-config-operator] adding namespace took 7.463588ms I0521 19:38:15.231393 3643 namespace.go:98] [openshift-monitoring] adding namespace I0521 19:38:15.231970 3643 namespace.go:102] [default] adding namespace took 7.642924ms I0521 19:38:15.231982 3643 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0521 19:38:15.232441 3643 namespace.go:102] [kube-public] adding namespace took 7.710291ms I0521 19:38:15.232450 3643 namespace.go:98] [openshift-backplane-tam] adding namespace I0521 19:38:15.232994 3643 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.796046ms I0521 19:38:15.233005 3643 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0521 19:38:15.233463 3643 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.815211ms I0521 19:38:15.233484 3643 namespace.go:98] [openshift-machine-api] adding namespace I0521 19:38:15.234016 3643 namespace.go:102] [openshift-authentication] adding namespace took 7.878508ms I0521 19:38:15.234030 3643 namespace.go:98] [openshift-service-ca-operator] adding namespace I0521 19:38:15.234503 3643 namespace.go:102] [openshift-backplane-ai-agent] adding namespace took 7.926037ms I0521 19:38:15.234519 3643 namespace.go:98] [openshift-must-gather-operator] adding namespace I0521 19:38:15.235019 3643 namespace.go:102] [openshift-console-operator] adding namespace took 7.67256ms I0521 19:38:15.235041 3643 namespace.go:98] [openshift-operators-redhat] adding namespace I0521 19:38:15.235672 3643 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.818461ms I0521 19:38:15.235688 3643 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0521 19:38:15.236602 3643 namespace.go:102] [kube-node-lease] adding namespace took 8.324485ms I0521 19:38:15.236625 3643 namespace.go:98] [openshift-customer-monitoring] adding namespace W0521 19:38:15.236603 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-m5pbz: pod openshift-network-diagnostics/network-check-target-m5pbz: no pod IPs found I0521 19:38:15.237217 3643 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.427054ms I0521 19:38:15.237231 3643 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0521 19:38:15.237662 3643 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 8.335943ms I0521 19:38:15.237678 3643 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0521 19:38:15.238142 3643 namespace.go:102] [openshift] adding namespace took 8.259554ms I0521 19:38:15.238158 3643 namespace.go:98] [openshift-backplane-csm] adding namespace I0521 19:38:15.238733 3643 namespace.go:102] [openshift-controller-manager] adding namespace took 8.281034ms I0521 19:38:15.238749 3643 namespace.go:98] [openshift-ingress-operator] adding namespace I0521 19:38:15.239323 3643 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.451164ms I0521 19:38:15.239342 3643 namespace.go:98] [openshift-console-user-settings] adding namespace I0521 19:38:15.239595 3643 default_node_network_controller.go:519] Node connection status = connected I0521 19:38:15.239797 3643 namespace.go:102] [openshift-monitoring] adding namespace took 8.398609ms I0521 19:38:15.239810 3643 namespace.go:98] [openshift-operators] adding namespace I0521 19:38:15.240455 3643 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 8.466258ms I0521 19:38:15.240473 3643 namespace.go:98] [openshift-etcd] adding namespace I0521 19:38:15.240929 3643 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.472513ms I0521 19:38:15.240945 3643 namespace.go:98] [openshift-config-managed] adding namespace I0521 19:38:15.241519 3643 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 8.508202ms I0521 19:38:15.241533 3643 namespace.go:98] [openshift-route-controller-manager] adding namespace I0521 19:38:15.242029 3643 namespace.go:102] [openshift-machine-api] adding namespace took 8.538919ms I0521 19:38:15.242626 3643 namespace.go:102] [openshift-service-ca-operator] adding namespace took 8.591512ms I0521 19:38:15.242635 3643 namespace.go:98] [openshift-cluster-version] adding namespace I0521 19:38:15.243092 3643 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.566763ms I0521 19:38:15.243108 3643 namespace.go:98] [openshift-network-console] adding namespace I0521 19:38:15.243624 3643 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.576325ms I0521 19:38:15.243637 3643 namespace.go:98] [openshift-kube-apiserver] adding namespace I0521 19:38:15.244127 3643 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.433453ms I0521 19:38:15.244138 3643 namespace.go:98] [openshift-host-network] adding namespace I0521 19:38:15.244553 3643 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.923088ms I0521 19:38:15.245044 3643 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.808873ms I0521 19:38:15.245632 3643 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.949358ms I0521 19:38:15.246154 3643 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.99112ms I0521 19:38:15.246669 3643 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.915248ms I0521 19:38:15.247170 3643 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.82161ms I0521 19:38:15.247185 3643 namespace.go:98] [openshift-infra] adding namespace I0521 19:38:15.247588 3643 namespace.go:102] [openshift-operators] adding namespace took 7.773203ms I0521 19:38:15.247600 3643 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0521 19:38:15.248108 3643 namespace.go:102] [openshift-etcd] adding namespace took 7.628709ms I0521 19:38:15.248581 3643 namespace.go:102] [openshift-config-managed] adding namespace took 7.63126ms I0521 19:38:15.249016 3643 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.477697ms I0521 19:38:15.249029 3643 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0521 19:38:15.249457 3643 namespace.go:102] [openshift-cluster-version] adding namespace took 6.817991ms I0521 19:38:15.249924 3643 namespace.go:102] [openshift-network-console] adding namespace took 6.808458ms I0521 19:38:15.250421 3643 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.777941ms I0521 19:38:15.250432 3643 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0521 19:38:15.250930 3643 namespace.go:102] [openshift-host-network] adding namespace took 6.785895ms I0521 19:38:15.251382 3643 namespace.go:102] [openshift-infra] adding namespace took 4.192269ms I0521 19:38:15.251791 3643 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 4.184993ms I0521 19:38:15.252313 3643 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 3.27891ms I0521 19:38:15.252664 3643 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 2.226575ms I0521 19:38:15.252857 3643 master.go:548] Adding or Updating local node "ip-10-0-2-167.ec2.internal" for network "default" I0521 19:38:15.253875 3643 zone_ic_handler.go:195] Time taken to create transit switch: 329.936µs I0521 19:38:15.253888 3643 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-26.ec2.internal for the network default E0521 19:38:15.254064 3643 obj_retry.go:575] Failed to create *v1.Node ip-10-0-0-62.ec2.internal, error: adding or updating remote node IC resources ip-10-0-0-62.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-0-62.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc006cd15b8 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u2062527157}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0521 19:38:15.254148 3643 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-62.ec2.internal", UID:"edeacc75-ed97-496f-8637-6bf325d8ecc2", APIVersion:"v1", ResourceVersion:"8933", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-0-62.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-0-62.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc006cd15b8 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u2062527157}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0521 19:38:15.254972 3643 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-26.ec2.internal" on network "default" took: 1.444483ms W0521 19:38:15.261934 3643 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-88jtx: pod kube-system/global-pull-secret-syncer-88jtx: no pod IPs found W0521 19:38:15.261953 3643 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-sc58z: pod openshift-multus/network-metrics-daemon-sc58z: no pod IPs found W0521 19:38:15.261961 3643 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-m5pbz: pod openshift-network-diagnostics/network-check-target-m5pbz: no pod IPs found I0521 19:38:15.263239 3643 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-167.ec2.internal for the network default I0521 19:38:15.263401 3643 zone_ic_handler.go:195] Time taken to create transit switch: 148.317µs I0521 19:38:15.264118 3643 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0521 19:38:15.264146 3643 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0521 19:38:15.264164 3643 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0521 19:38:15.264167 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-62.ec2.internal in network "default" I0521 19:38:15.264235 3643 node_tracker.go:169] Node ip-10-0-0-62.ec2.internal switch + router changed, syncing services in network "default" I0521 19:38:15.264254 3643 services_controller.go:574] Full service sync requested for network=default I0521 19:38:15.264264 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-26.ec2.internal in network "default" I0521 19:38:15.264291 3643 node_tracker.go:169] Node ip-10-0-1-26.ec2.internal switch + router changed, syncing services in network "default" I0521 19:38:15.264298 3643 services_controller.go:574] Full service sync requested for network=default I0521 19:38:15.264305 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-167.ec2.internal in network "default" I0521 19:38:15.264324 3643 node_tracker.go:169] Node ip-10-0-2-167.ec2.internal switch + router changed, syncing services in network "default" I0521 19:38:15.264330 3643 services_controller.go:574] Full service sync requested for network=default I0521 19:38:15.364628 3643 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0521 19:38:15.364653 3643 services_controller.go:208] Setting up event handlers for services for network=default I0521 19:38:15.364723 3643 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0521 19:38:15.364780 3643 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0521 19:38:15.364797 3643 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0521 19:38:15.465253 3643 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0521 19:38:15.465274 3643 repair.go:60] Starting repairing loop for services I0521 19:38:15.465356 3643 repair.go:131] Deleted 0 stale service LBs I0521 19:38:15.465375 3643 repair.go:137] Deleted 0 stale Chassis Template Vars I0521 19:38:15.465392 3643 repair.go:62] Finished repairing loop for services: 121.33µs I0521 19:38:15.465415 3643 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0521 19:38:15.465426 3643 services_controller.go:254] Starting workers for network=default I0521 19:38:15.466176 3643 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-sc58z] creating logical port openshift-multus_network-metrics-daemon-sc58z for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:15.466298 3643 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-m5pbz] creating logical port openshift-network-diagnostics_network-check-target-m5pbz for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:15.466311 3643 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-sc58z I0521 19:38:15.466401 3643 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-m5pbz I0521 19:38:15.466604 3643 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-88jtx] creating logical port kube-system_global-pull-secret-syncer-88jtx for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:15.466713 3643 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-88jtx I0521 19:38:15.483870 3643 pod.go:62] [kube-system/global-pull-secret-syncer-88jtx] pod update took 17.172234ms I0521 19:38:15.483891 3643 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-88jtx] addLogicalPort annotation time took 17.195514ms I0521 19:38:15.484752 3643 pods.go:271] [kube-system/global-pull-secret-syncer-88jtx] addLogicalPort took 18.154606ms, libovsdb time 610.614µs I0521 19:38:15.485356 3643 pod.go:62] [openshift-network-diagnostics/network-check-target-m5pbz] pod update took 18.989689ms I0521 19:38:15.485375 3643 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-m5pbz] addLogicalPort annotation time took 19.008028ms I0521 19:38:15.485436 3643 pod.go:62] [openshift-multus/network-metrics-daemon-sc58z] pod update took 19.180679ms I0521 19:38:15.485455 3643 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-sc58z] addLogicalPort annotation time took 19.203515ms I0521 19:38:15.486260 3643 pods.go:271] [openshift-network-diagnostics/network-check-target-m5pbz] addLogicalPort took 19.971854ms, libovsdb time 655.802µs I0521 19:38:15.486569 3643 pods.go:271] [openshift-multus/network-metrics-daemon-sc58z] addLogicalPort took 20.405075ms, libovsdb time 885.151µs I0521 19:38:15.487015 3643 admin_network_policy_controller.go:227] Starting controller default-network-controller I0521 19:38:15.487037 3643 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0521 19:38:15.487053 3643 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0521 19:38:15.487062 3643 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0521 19:38:15.487277 3643 repair.go:29] Repairing admin network policies took 207.102µs I0521 19:38:15.487418 3643 repair.go:92] Repairing baseline admin network policies took 129.058µs I0521 19:38:15.487935 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0521 19:38:15.487953 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0521 19:38:15.489746 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0521 19:38:15.489766 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0521 19:38:15.489775 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0521 19:38:15.490267 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0521 19:38:15.490286 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0521 19:38:15.490295 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0521 19:38:15.492179 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0521 19:38:15.492198 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0521 19:38:15.492206 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0521 19:38:15.493718 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0521 19:38:15.493741 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0521 19:38:15.493749 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0521 19:38:15.494297 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0521 19:38:15.494322 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0521 19:38:15.494333 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0521 19:38:15.494840 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0521 19:38:15.494859 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0521 19:38:15.494867 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0521 19:38:15.495362 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0521 19:38:15.495379 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0521 19:38:15.495386 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0521 19:38:15.495639 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0521 19:38:15.495656 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0521 19:38:15.495664 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0521 19:38:15.496960 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0521 19:38:15.496976 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0521 19:38:15.496985 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0521 19:38:15.497589 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0521 19:38:15.497615 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0521 19:38:15.498048 3643 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0521 19:38:15.498062 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0521 19:38:15.498729 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0521 19:38:15.498752 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0521 19:38:15.498765 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0521 19:38:15.500229 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0521 19:38:15.500249 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0521 19:38:15.500261 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0521 19:38:15.501513 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0521 19:38:15.501537 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0521 19:38:15.501559 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0521 19:38:15.501835 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0521 19:38:15.501854 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0521 19:38:15.501882 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0521 19:38:15.502691 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0521 19:38:15.502711 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0521 19:38:15.502725 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0521 19:38:15.503271 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0521 19:38:15.503292 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0521 19:38:15.503302 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0521 19:38:15.503815 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0521 19:38:15.503837 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0521 19:38:15.503844 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0521 19:38:15.505238 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0521 19:38:15.505256 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0521 19:38:15.505264 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0521 19:38:15.506150 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0521 19:38:15.506169 3643 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0521 19:38:15.506179 3643 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0521 19:38:15.508188 3643 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0521 19:38:15.508207 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0521 19:38:15.508216 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0521 19:38:15.509493 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0521 19:38:15.509511 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0521 19:38:15.509519 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0521 19:38:15.510040 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0521 19:38:15.510063 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0521 19:38:15.510073 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0521 19:38:15.511213 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0521 19:38:15.511233 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0521 19:38:15.511246 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0521 19:38:15.512373 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0521 19:38:15.512391 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0521 19:38:15.512399 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0521 19:38:15.513471 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0521 19:38:15.513490 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0521 19:38:15.513497 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0521 19:38:15.514020 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0521 19:38:15.514042 3643 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0521 19:38:15.514048 3643 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0521 19:38:15.514354 3643 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0521 19:38:15.514370 3643 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0521 19:38:15.514385 3643 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0521 19:38:15.515317 3643 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0521 19:38:15.515334 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0521 19:38:15.515345 3643 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0521 19:38:15.517145 3643 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0521 19:38:15.523186 3643 egressfirewall.go:374] Starting EgressFirewall controller I0521 19:38:15.523227 3643 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0521 19:38:15.523226 3643 controller.go:133] Adding controller egress-firewall-controller event handlers I0521 19:38:15.523257 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0521 19:38:15.523271 3643 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0521 19:38:15.523270 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0521 19:38:15.523281 3643 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0521 19:38:15.523564 3643 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0521 19:38:15.523579 3643 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0521 19:38:15.523586 3643 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0521 19:38:15.523597 3643 egressqos.go:193] Setting up event handlers for EgressQoS I0521 19:38:15.523610 3643 egressfirewall.go:723] Syncing node "ip-10-0-0-62.ec2.internal" for egress firewall I0521 19:38:15.523622 3643 egressfirewall.go:723] Syncing node "ip-10-0-1-26.ec2.internal" for egress firewall I0521 19:38:15.523631 3643 egressfirewall.go:723] Syncing node "ip-10-0-2-167.ec2.internal" for egress firewall I0521 19:38:15.523862 3643 egressqos.go:245] Starting EgressQoS Controller I0521 19:38:15.523877 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0521 19:38:15.523886 3643 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0521 19:38:15.523891 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0521 19:38:15.523896 3643 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0521 19:38:15.523917 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0521 19:38:15.523923 3643 shared_informer.go:356] "Caches are synced" controller="egressqos" I0521 19:38:15.523927 3643 egressqos.go:259] Repairing EgressQoSes I0521 19:38:15.523931 3643 egressqos.go:399] Starting repairing loop for egressqos I0521 19:38:15.524006 3643 egressqos.go:401] Finished repairing loop for egressqos: 74.763µs I0521 19:38:15.524032 3643 egressservice_zone.go:132] Setting up event handlers for Egress Services I0521 19:38:15.524071 3643 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-167.ec2.internal I0521 19:38:15.524147 3643 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-167.ec2.internal : 76.814µs I0521 19:38:15.524317 3643 egressservice_zone.go:210] Starting Egress Services Controller I0521 19:38:15.524325 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0521 19:38:15.524331 3643 shared_informer.go:356] "Caches are synced" controller="egressservices" I0521 19:38:15.524337 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0521 19:38:15.524341 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0521 19:38:15.524346 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0521 19:38:15.524364 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0521 19:38:15.524371 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0521 19:38:15.524375 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0521 19:38:15.524379 3643 egressservice_zone.go:228] Repairing Egress Services I0521 19:38:15.524725 3643 master_controller.go:88] Starting Admin Policy Based Route Controller I0521 19:38:15.524735 3643 external_controller.go:278] Starting Admin Policy Based Route Controller I0521 19:38:15.524773 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-62.ec2.internal I0521 19:38:15.524785 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-62.ec2.internal: 13.918µs I0521 19:38:15.524792 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-26.ec2.internal I0521 19:38:15.524796 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-26.ec2.internal: 3.975µs I0521 19:38:15.524801 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-167.ec2.internal I0521 19:38:15.524822 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-167.ec2.internal: 4.211µs I0521 19:38:15.524988 3643 default_network_controller.go:585] Completing all the Watchers took 314.994499ms I0521 19:38:15.525001 3643 default_network_controller.go:589] Starting unidling controllers I0521 19:38:15.525012 3643 unidle.go:46] Registering OVN SB ControllerEvent handler I0521 19:38:15.525017 3643 unidle.go:63] Populating Initial ContollerEvent events I0521 19:38:15.525033 3643 unidle.go:79] Setting up event handlers for services I0521 19:38:15.525153 3643 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0521 19:38:15.740176 3643 default_node_network_controller.go:519] Node connection status = connected I0521 19:38:16.033480 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-26.ec2.internal I0521 19:38:16.033532 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-26.ec2.internal: 24.654µs I0521 19:38:16.033554 3643 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-26.ec2.internal. Updating no re-route policies I0521 19:38:16.033568 3643 egressfirewall.go:723] Syncing node "ip-10-0-1-26.ec2.internal" for egress firewall I0521 19:38:16.033554 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-26.ec2.internal in network "default" I0521 19:38:16.033673 3643 node_tracker.go:169] Node ip-10-0-1-26.ec2.internal switch + router changed, syncing services in network "default" I0521 19:38:16.033688 3643 services_controller.go:574] Full service sync requested for network=default I0521 19:38:16.240528 3643 default_node_network_controller.go:519] Node connection status = connected I0521 19:38:16.740013 3643 default_node_network_controller.go:519] Node connection status = connected I0521 19:38:17.239651 3643 default_node_network_controller.go:519] Node connection status = connected I0521 19:38:17.661382 3643 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0521 19:38:17.697039 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-62.ec2.internal I0521 19:38:17.697074 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-62.ec2.internal: 44.413µs I0521 19:38:17.697111 3643 default_network_controller.go:1025] Node "ip-10-0-0-62.ec2.internal" in remote zone "ip-10-0-0-62.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:38:17.697878 3643 zone_ic_handler.go:195] Time taken to create transit switch: 274.125µs I0521 19:38:17.697890 3643 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-62.ec2.internal for the network default I0521 19:38:17.698859 3643 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-62.ec2.internal" on network "default" took: 1.266103ms I0521 19:38:17.739811 3643 default_node_network_controller.go:519] Node connection status = connected I0521 19:38:17.758422 3643 bridgeconfig.go:537] Gateway is ready I0521 19:38:17.758441 3643 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0521 19:38:17.770410 3643 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.167. Updating OVN encap IP. I0521 19:38:17.775801 3643 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.167 - it is already configured I0521 19:38:17.775820 3643 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.167/24:{} 172.20.0.1/32:{}]. Updating annotations. I0521 19:38:17.776107 3643 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.167/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-167.ec2.internal","mac-address":"0e:a3:e7:4e:db:71","ip-addresses":["10.0.2.167/24"],"ip-address":"10.0.2.167/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:fb42eae9-0346-4884-94e5-86d8ea9cc637 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.167/24"}] on node ip-10-0-2-167.ec2.internal I0521 19:38:17.800448 3643 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.167 I0521 19:38:17.800556 3643 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-167.ec2.internal I0521 19:38:17.801022 3643 egressfirewall.go:723] Syncing node "ip-10-0-2-167.ec2.internal" for egress firewall I0521 19:38:17.801055 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-167.ec2.internal I0521 19:38:17.801062 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-167.ec2.internal in network "default" I0521 19:38:17.801069 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-167.ec2.internal: 19.622µs I0521 19:38:17.801080 3643 master.go:548] Adding or Updating local node "ip-10-0-2-167.ec2.internal" for network "default" I0521 19:38:17.801109 3643 node_tracker.go:169] Node ip-10-0-2-167.ec2.internal switch + router changed, syncing services in network "default" I0521 19:38:17.801125 3643 services_controller.go:574] Full service sync requested for network=default I0521 19:38:17.806202 3643 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-167.ec2.internal. Updating no re-route policies I0521 19:38:17.806274 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-sc58z I0521 19:38:17.806286 3643 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-88jtx I0521 19:38:17.806296 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-2lnzv I0521 19:38:17.806302 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-q497d I0521 19:38:17.806305 3643 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-88jtx I0521 19:38:17.806304 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-8csd2 I0521 19:38:17.806311 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-q497d I0521 19:38:17.806312 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-2lnzv I0521 19:38:17.806314 3643 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-qwqt8 I0521 19:38:17.806321 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-8csd2 I0521 19:38:17.806334 3643 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-qwqt8 I0521 19:38:17.806335 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-2lnzv after 0 failed attempt(s) I0521 19:38:17.806334 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-jlnxj I0521 19:38:17.806343 3643 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-88jtx] creating logical port kube-system_global-pull-secret-syncer-88jtx for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:17.806342 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-n285q I0521 19:38:17.806347 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-167.ec2.internal I0521 19:38:17.806351 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-8csd2 after 0 failed attempt(s) I0521 19:38:17.806354 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-4nsjr I0521 19:38:17.806358 3643 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-qwqt8 after 0 failed attempt(s) I0521 19:38:17.806359 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-n285q I0521 19:38:17.806361 3643 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-167.ec2.internal I0521 19:38:17.806364 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-jqm6r I0521 19:38:17.806370 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-46mzj I0521 19:38:17.806372 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-jqm6r I0521 19:38:17.806372 3643 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-167.ec2.internal I0521 19:38:17.806380 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-46mzj I0521 19:38:17.806383 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-n285q after 0 failed attempt(s) I0521 19:38:17.806390 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-jqm6r after 0 failed attempt(s) I0521 19:38:17.806335 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-q497d after 0 failed attempt(s) I0521 19:38:17.806394 3643 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-167.ec2.internal after 0 failed attempt(s) I0521 19:38:17.806356 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-167.ec2.internal I0521 19:38:17.806398 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-46mzj after 0 failed attempt(s) I0521 19:38:17.806362 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-4nsjr I0521 19:38:17.806415 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-167.ec2.internal after 0 failed attempt(s) I0521 19:38:17.806427 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-4nsjr after 0 failed attempt(s) I0521 19:38:17.806294 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-sc58z I0521 19:38:17.806467 3643 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-sc58z] creating logical port openshift-multus_network-metrics-daemon-sc58z for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:17.806348 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-m5pbz I0521 19:38:17.806483 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-m5pbz I0521 19:38:17.806516 3643 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-m5pbz] creating logical port openshift-network-diagnostics_network-check-target-m5pbz for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:17.806349 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-jlnxj I0521 19:38:17.806578 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-jlnxj after 0 failed attempt(s) I0521 19:38:17.807219 3643 pods.go:271] [kube-system/global-pull-secret-syncer-88jtx] addLogicalPort took 884.292µs, libovsdb time 317.954µs I0521 19:38:17.807235 3643 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-88jtx after 0 failed attempt(s) I0521 19:38:17.807292 3643 pods.go:271] [openshift-network-diagnostics/network-check-target-m5pbz] addLogicalPort took 786.299µs, libovsdb time 260.875µs I0521 19:38:17.807305 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-m5pbz after 0 failed attempt(s) I0521 19:38:17.807367 3643 pods.go:271] [openshift-multus/network-metrics-daemon-sc58z] addLogicalPort took 910.636µs, libovsdb time 375.825µs I0521 19:38:17.807384 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-sc58z after 0 failed attempt(s) I0521 19:38:17.824946 3643 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0521 19:38:17.837323 3643 iptables.go:108] Creating table: mangle chain: OUTPUT I0521 19:38:17.840422 3643 iptables.go:108] Creating table: nat chain: OUTPUT I0521 19:38:17.871596 3643 iptables.go:108] Creating table: nat chain: PREROUTING I0521 19:38:17.918477 3643 iptables.go:108] Creating table: filter chain: FORWARD I0521 19:38:17.934706 3643 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0521 19:38:17.934816 3643 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0521 19:38:17.934865 3643 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0521 19:38:17.934979 3643 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0521 19:38:17.935053 3643 gateway_init.go:325] Gateway and management port readiness took 2.700039478s I0521 19:38:17.935087 3643 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0521 19:38:17.935133 3643 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0521 19:38:17.935150 3643 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0521 19:38:17.935162 3643 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0521 19:38:17.935239 3643 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0521 19:38:17.935319 3643 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 55.635µs I0521 19:38:17.935722 3643 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0521 19:38:17.935755 3643 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 21.783µs I0521 19:38:17.935764 3643 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 45.355µs I0521 19:38:17.935770 3643 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0521 19:38:17.935779 3643 nad_controller.go:379] [node-nad-controller NAD controller]: started I0521 19:38:17.935800 3643 default_node_network_controller.go:899] Starting the default node network controller I0521 19:38:17.935807 3643 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0521 19:38:17.935810 3643 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.167µs I0521 19:38:17.936071 3643 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0521 19:38:17.936084 3643 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0521 19:38:17.936118 3643 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000329070 0xc00019b9d0 0xc00019ba40 0xc00019bab0 0xc00019bb20 0xc00019bb90 0xc00019bc00 0xc000325450 0xc00019bce0 0xc00019bd50 0xc00020a050 0xc006566620 0xc00019bdc0 map[0x2ff9080:0xc00290a660 0x2ff95c0:0xc0049194a0 0x2ffa2e0:0xc00290a5a0 0x2ffa580:0xc00290a600 0x2ffad60:0xc00290a6c0 0x2ffb000:0xc00290a720 0x2ffb2a0:0xc004919440 0x2ffba80:0xc004919500 0x2ffbd20:0xc004919560 0x3026980:0xc00118ac60 0x3026d20:0xc00118acc0 0x3027800:0xc000d253e0 0x3029160:0xc001b68000 0x30298a0:0xc00118aea0 0x302a720:0xc001b68060] 0xc000d061c0 0} I0521 19:38:17.936173 3643 gateway.go:151] Starting gateway service sync I0521 19:38:17.936718 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0521 19:38:17.938877 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0521 19:38:17.948835 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0521 19:38:17.950066 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0521 19:38:17.951247 3643 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0521 19:38:17.985154 3643 gateway.go:168] Gateway service sync done. Time taken: 48.967998ms I0521 19:38:17.985759 3643 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31070 I0521 19:38:17.986030 3643 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000329070 0xc00019b9d0 0xc00019ba40 0xc00019bab0 0xc00019bb20 0xc00019bb90 0xc00019bc00 0xc000325450 0xc00019bce0 0xc00019bd50 0xc00020a050 0xc006566620 0xc00019bdc0 map[0x2ff9080:0xc00290a660 0x2ff95c0:0xc0049194a0 0x2ffa2e0:0xc00290a5a0 0x2ffa580:0xc00290a600 0x2ffad60:0xc00290a6c0 0x2ffb000:0xc00290a720 0x2ffb2a0:0xc004919440 0x2ffba80:0xc004919500 0x2ffbd20:0xc004919560 0x3026980:0xc00118ac60 0x3026d20:0xc00118acc0 0x3027800:0xc000d253e0 0x3029160:0xc001b68000 0x30298a0:0xc00118aea0 0x302a720:0xc001b68060] 0xc000d061c0 0} I0521 19:38:17.986464 3643 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000329070 0xc00019b9d0 0xc00019ba40 0xc00019bab0 0xc00019bb20 0xc00019bb90 0xc00019bc00 0xc000325450 0xc00019bce0 0xc00019bd50 0xc00020a050 0xc006566620 0xc00019bdc0 map[0x2ff9080:0xc00290a660 0x2ff95c0:0xc0049194a0 0x2ffa2e0:0xc00290a5a0 0x2ffa580:0xc00290a600 0x2ffad60:0xc00290a6c0 0x2ffb000:0xc00290a720 0x2ffb2a0:0xc004919440 0x2ffba80:0xc004919500 0x2ffbd20:0xc004919560 0x3026980:0xc00118ac60 0x3026d20:0xc00118acc0 0x3027800:0xc000d253e0 0x3029160:0xc001b68000 0x30298a0:0xc00118aea0 0x302a720:0xc001b68060] 0xc000d061c0 0} I0521 19:38:17.986500 3643 gateway.go:532] Reconciling gateway with updates I0521 19:38:17.992100 3643 gateway.go:339] Spawning Conntrack Rule Check Thread I0521 19:38:17.992172 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:38:17.992190 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:38:17.992198 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:38:17.992210 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0521 19:38:17.992214 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:38:17.992223 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0521 19:38:17.992219 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:38:17.992232 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0521 19:38:17.992240 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:38:17.992246 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0521 19:38:17.992246 3643 default_node_network_controller.go:944] Gateway and management port readiness took 56.151064ms I0521 19:38:17.992249 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:38:17.992261 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0521 19:38:17.992262 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:38:17.992269 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0521 19:38:17.992276 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:38:17.992277 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:38:17.992285 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:38:17.992287 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:38:17.992293 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0521 19:38:17.992299 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0521 19:38:17.992196 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:38:17.992309 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:38:17.992314 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:38:17.992318 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0521 19:38:17.992329 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:38:17.992330 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0521 19:38:17.992334 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0521 19:38:17.992331 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:38:17.992328 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:38:17.992342 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:38:17.992344 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:38:17.992346 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0521 19:38:17.992351 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:38:17.992354 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0521 19:38:17.992318 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:38:17.992362 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0521 19:38:17.992364 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:38:17.992372 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:38:17.992378 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:38:17.992381 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:38:17.992387 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0521 19:38:17.992388 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:38:17.992293 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:38:17.992374 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0521 19:38:17.992406 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:38:17.992415 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0521 19:38:17.992417 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0521 19:38:17.992314 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:38:17.992423 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0521 19:38:17.992436 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0521 19:38:17.992350 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:38:17.992451 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0521 19:38:17.992275 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:38:17.992462 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:38:17.992467 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0521 19:38:17.992296 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:38:17.992478 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0521 19:38:17.992282 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:38:17.992499 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:38:17.992500 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:38:17.992507 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0521 19:38:17.992304 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0521 19:38:17.992530 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0521 19:38:17.992501 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0521 19:38:17.992542 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0521 19:38:17.992262 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 31985 is already open, no action needed I0521 19:38:17.992560 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 31839 is already open, no action needed I0521 19:38:17.992559 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0521 19:38:17.992585 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:38:17.992301 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:38:17.992596 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:38:17.992601 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:38:17.992608 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0521 19:38:17.992305 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:38:17.992618 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:38:17.992628 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:38:17.992319 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:38:17.992666 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:38:17.992668 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:38:17.992678 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:38:17.992323 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:38:17.992722 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:38:17.992738 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:38:17.992322 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:38:17.992753 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:38:17.992783 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:38:17.992802 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:38:17.992820 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:38:17.992843 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:38:17.992867 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:38:17.992890 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:38:17.992330 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:38:17.992889 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:38:17.992922 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:38:17.992931 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0521 19:38:17.992336 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:38:17.992937 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:38:17.992349 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:38:17.992953 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:38:17.992363 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:38:17.992942 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:38:17.993006 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:38:17.992369 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:38:17.993059 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:38:17.992303 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0521 19:38:17.992514 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:38:17.993099 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0521 19:38:17.992654 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0521 19:38:17.993122 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0521 19:38:17.992357 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0521 19:38:17.993191 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0521 19:38:17.992499 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:38:17.993210 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:38:17.993851 3643 node_ip_handler_linux.go:160] Node IP manager is running W0521 19:38:18.033760 3643 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0521 19:38:18.033941 3643 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0521 19:38:18.034083 3643 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0521 19:38:18.043299 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0521 19:38:18.065230 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0521 19:38:18.065246 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0521 19:38:18.065234 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0521 19:38:18.065263 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0521 19:38:18.065271 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0521 19:38:18.065276 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0521 19:38:18.065281 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0521 19:38:18.065287 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0521 19:38:18.065293 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0521 19:38:18.065298 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0521 19:38:18.065304 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0521 19:38:18.065309 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0521 19:38:18.065314 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0521 19:38:18.065320 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0521 19:38:18.065325 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0521 19:38:18.065330 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0521 19:38:18.065335 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0521 19:38:18.065341 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0521 19:38:18.106761 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0521 19:38:18.106790 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0521 19:38:19.845510 3643 default_node_network_controller.go:1380] Starting node controller node sync W0521 19:38:19.845514 3643 egressip_healthcheck.go:76] Health checking using insecure connection I0521 19:38:19.845648 3643 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0521 19:38:19.857199 3643 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.665809ms I0521 19:38:19.857350 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-62.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0521 19:38:19.857366 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-62.ec2.internal", IP: 10.0.0.62 to PMTUD blocking rules I0521 19:38:19.857352 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-26.ec2.internal", IP: 10.0.1.26 to PMTUD blocking rules I0521 19:38:19.857414 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-26.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0521 19:38:19.869009 3643 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0521 19:38:19.869289 3643 egressservice_node.go:93] Setting up event handlers for Egress Services I0521 19:38:19.869420 3643 egressservice_node.go:183] Starting Egress Services Controller I0521 19:38:19.869439 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0521 19:38:19.869451 3643 shared_informer.go:356] "Caches are synced" controller="egressservices" I0521 19:38:19.869461 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0521 19:38:19.869470 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0521 19:38:19.869478 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0521 19:38:19.869484 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0521 19:38:19.869488 3643 egressservice_node.go:197] Repairing Egress Services I0521 19:38:19.879575 3643 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0521 19:38:19.879587 3643 external_controller.go:278] Starting Admin Policy Based Route Controller I0521 19:38:19.879717 3643 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0521 19:38:19.879731 3643 link_network_manager.go:119] Link manager is running I0521 19:38:19.879739 3643 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0521 19:38:19.879919 3643 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0521 19:38:19.880480 3643 vrf_manager.go:128] VRF manager is running I0521 19:38:19.880822 3643 node_controller_manager.go:437] Removing flows to drop GARP I0521 19:38:19.880833 3643 gateway.go:532] Reconciling gateway with updates I0521 19:38:19.910145 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:38:19.910163 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0521 19:38:19.910164 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:38:19.910174 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0521 19:38:19.910177 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:38:19.910181 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:38:19.910189 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:38:19.910196 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:38:19.910200 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:38:19.910203 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:38:19.910199 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:38:19.910206 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0521 19:38:19.910212 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:38:19.910213 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:38:19.910215 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0521 19:38:19.910223 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:38:19.910224 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:38:19.910228 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:38:19.910236 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:38:19.910174 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:38:19.910250 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0521 19:38:19.910256 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:38:19.910262 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:38:19.910263 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:38:19.910273 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:38:19.910276 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:38:19.910274 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:38:19.910280 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0521 19:38:19.910232 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0521 19:38:19.910288 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:38:19.910292 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0521 19:38:19.910294 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:38:19.910300 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:38:19.910300 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:38:19.910304 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:38:19.910309 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0521 19:38:19.910307 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:38:19.910314 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:38:19.910168 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:38:19.910320 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:38:19.910322 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:38:19.910322 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:38:19.910329 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0521 19:38:19.910334 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:38:19.910334 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:38:19.910338 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:38:19.910336 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:38:19.910353 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:38:19.910361 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0521 19:38:19.910185 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:38:19.910257 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:38:19.910342 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0521 19:38:19.910274 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:38:19.910224 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:38:19.910510 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:38:19.910324 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:38:19.910535 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0521 19:38:19.910315 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:38:19.910311 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:38:19.910196 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:38:19.910581 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:38:19.910209 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:38:19.910586 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0521 19:38:19.910595 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:38:19.910213 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:38:19.910668 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:38:19.910290 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:38:19.910187 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:38:19.910186 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:38:19.910775 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:38:19.910203 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:38:19.910150 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:38:19.910846 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:38:19.910852 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0521 19:38:19.910251 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:38:19.910275 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0521 19:38:19.910271 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0521 19:38:19.910899 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0521 19:38:19.910931 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 31985 is already open, no action needed I0521 19:38:19.910940 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 31839 is already open, no action needed I0521 19:38:19.910282 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:38:19.910291 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:38:19.911009 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:38:19.910303 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0521 19:38:19.910150 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:38:19.911035 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:38:19.911039 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0521 19:38:19.910304 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:38:19.910304 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:38:19.911077 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:38:19.910320 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0521 19:38:19.911103 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0521 19:38:19.910319 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:38:19.911133 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:38:19.910318 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:38:19.911157 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:38:19.911162 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0521 19:38:19.910323 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:38:19.910329 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:38:19.911570 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0521 19:38:19.911579 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0521 19:38:19.911590 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0521 19:38:19.911596 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0521 19:38:19.911602 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0521 19:38:19.911607 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0521 19:38:19.911612 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0521 19:38:19.911617 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0521 19:38:19.911621 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0521 19:38:19.911627 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0521 19:38:19.911632 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0521 19:38:19.911640 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0521 19:38:19.911652 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0521 19:38:19.911657 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0521 19:38:19.911662 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0521 19:38:19.911667 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0521 19:38:19.911671 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0521 19:38:19.911678 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0521 19:38:19.911685 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0521 19:38:19.911693 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0521 19:38:19.911698 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0521 19:38:19.911703 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0521 19:38:19.911707 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0521 19:38:19.911712 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0521 19:38:19.911717 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0521 19:38:19.953062 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0521 19:38:19.973644 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0521 19:38:19.973683 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0521 19:38:19.973700 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0521 19:38:19.973708 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0521 19:38:19.973714 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0521 19:38:19.973728 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0521 19:38:19.973754 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0521 19:38:19.973772 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0521 19:38:19.973779 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0521 19:38:20.132397 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-26.ec2.internal I0521 19:38:20.132427 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-26.ec2.internal: 34.646µs I0521 19:38:21.631437 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-167.ec2.internal I0521 19:38:21.631468 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-167.ec2.internal: 38.941µs I0521 19:38:21.660107 3643 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6778768bb7-g9rlf] creating logical port openshift-image-registry_image-registry-6778768bb7-g9rlf for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:21.660249 3643 kube.go:256] Updating pod openshift-image-registry/image-registry-6778768bb7-g9rlf I0521 19:38:21.683594 3643 pod.go:62] [openshift-image-registry/image-registry-6778768bb7-g9rlf] pod update took 23.376094ms I0521 19:38:21.683617 3643 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6778768bb7-g9rlf] addLogicalPort annotation time took 23.404681ms I0521 19:38:21.683620 3643 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-lfrcc] creating logical port openshift-dns_dns-default-lfrcc for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:21.683738 3643 kube.go:256] Updating pod openshift-dns/dns-default-lfrcc I0521 19:38:21.684880 3643 pods.go:271] [openshift-image-registry/image-registry-6778768bb7-g9rlf] addLogicalPort took 24.785893ms, libovsdb time 1.007941ms I0521 19:38:21.685060 3643 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-t4cqf] creating logical port openshift-ingress-canary_ingress-canary-t4cqf for pod on switch ip-10-0-2-167.ec2.internal I0521 19:38:21.685171 3643 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-t4cqf I0521 19:38:21.705691 3643 pod.go:62] [openshift-dns/dns-default-lfrcc] pod update took 21.979234ms I0521 19:38:21.705718 3643 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-lfrcc] addLogicalPort annotation time took 22.008617ms I0521 19:38:21.706920 3643 pods.go:271] [openshift-dns/dns-default-lfrcc] addLogicalPort took 23.296167ms, libovsdb time 840.02µs I0521 19:38:21.707258 3643 pod.go:62] [openshift-ingress-canary/ingress-canary-t4cqf] pod update took 22.115653ms I0521 19:38:21.707276 3643 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-t4cqf] addLogicalPort annotation time took 22.137176ms I0521 19:38:21.708244 3643 pods.go:271] [openshift-ingress-canary/ingress-canary-t4cqf] addLogicalPort took 23.194956ms, libovsdb time 744.949µs I0521 19:38:22.034686 3643 cni.go:452] [openshift-ingress-canary/ingress-canary-t4cqf ee975e5101e28f3d6534d04aa224beee4f5f9313e389e97329cd39abc4546b09 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-t4cqf ee975e5101e28f3d6534d04aa224beee4f5f9313e389e97329cd39abc4546b09 network default NAD default NAD key ] I0521 19:38:22.052139 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-t4cqf, hostIfaceName: ee975e5101e28f3, network: default, NAD default, SandboxID: "ee975e5101e28f3d6534d04aa224beee4f5f9313e389e97329cd39abc4546b09", PCI device ID: , UID: "8971fe73-542e-453d-8ac1-df387d659404", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0521 19:38:22.106022 3643 cni.go:473] [openshift-ingress-canary/ingress-canary-t4cqf ee975e5101e28f3d6534d04aa224beee4f5f9313e389e97329cd39abc4546b09 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-t4cqf ee975e5101e28f3d6534d04aa224beee4f5f9313e389e97329cd39abc4546b09 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:f7:21:b4:35:11\",\"name\":\"ee975e5101e28f3\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4eb6f7c2-eb4f-4809-99ff-1b8f838f6f4e\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:38:25.842597 3643 cni.go:452] [openshift-network-diagnostics/network-check-target-m5pbz fb5a0def5e9713c344d423c3c6c2920edf5d76e3d9265ea9075d68fe7dc67d70 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-m5pbz fb5a0def5e9713c344d423c3c6c2920edf5d76e3d9265ea9075d68fe7dc67d70 network default NAD default NAD key ] I0521 19:38:25.845720 3643 cni.go:452] [kube-system/global-pull-secret-syncer-88jtx 0829367cb93b8dcaf3bf0a70b73471a300ce561fb7d46b01641fbf967f3e6835 network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-88jtx 0829367cb93b8dcaf3bf0a70b73471a300ce561fb7d46b01641fbf967f3e6835 network default NAD default NAD key ] I0521 19:38:25.852778 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-m5pbz, hostIfaceName: fb5a0def5e9713c, network: default, NAD default, SandboxID: "fb5a0def5e9713c344d423c3c6c2920edf5d76e3d9265ea9075d68fe7dc67d70", PCI device ID: , UID: "fdeded3a-bf20-4ef1-a6e4-ab5636175ca1", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0521 19:38:25.855755 3643 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-88jtx, hostIfaceName: 0829367cb93b8dc, network: default, NAD default, SandboxID: "0829367cb93b8dcaf3bf0a70b73471a300ce561fb7d46b01641fbf967f3e6835", PCI device ID: , UID: "ab419b54-aa31-4c74-8996-7bfbb27f6f48", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0521 19:38:25.900164 3643 cni.go:473] [openshift-network-diagnostics/network-check-target-m5pbz fb5a0def5e9713c344d423c3c6c2920edf5d76e3d9265ea9075d68fe7dc67d70 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-m5pbz fb5a0def5e9713c344d423c3c6c2920edf5d76e3d9265ea9075d68fe7dc67d70 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:45:d5:d3:6e:12\",\"name\":\"fb5a0def5e9713c\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ed17d1f-6683-451b-ab63-68eb2fb5e34d\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:38:25.905536 3643 cni.go:473] [kube-system/global-pull-secret-syncer-88jtx 0829367cb93b8dcaf3bf0a70b73471a300ce561fb7d46b01641fbf967f3e6835 network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-88jtx 0829367cb93b8dcaf3bf0a70b73471a300ce561fb7d46b01641fbf967f3e6835 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:be:99:6b:72:57\",\"name\":\"0829367cb93b8dc\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ec5ec86-761c-4b0f-adc0-3b25edcdb063\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:38:27.606153 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0521 19:38:27.655674 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:38:29.932411 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:38:30.958150 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:38:58.608151 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:38:58.608174 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:39:00.929982 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:39:00.930006 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:39:01.947262 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0521 19:40:07.803735 3643 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-9xbfp for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:07.803826 3643 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp I0521 19:40:07.823201 3643 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-b577n] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-b577n for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:07.823287 3643 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-b577n I0521 19:40:07.825991 3643 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp] pod update took 22.190912ms I0521 19:40:07.826014 3643 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp] addLogicalPort annotation time took 22.218303ms I0521 19:40:07.827205 3643 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp] addLogicalPort took 23.486628ms, libovsdb time 908.141µs I0521 19:40:07.849236 3643 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-b577n] pod update took 25.974647ms I0521 19:40:07.849257 3643 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-b577n] addLogicalPort annotation time took 26.000919ms I0521 19:40:07.850146 3643 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-b577n] addLogicalPort took 26.956207ms, libovsdb time 655.948µs I0521 19:40:07.937017 3643 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-zmprk for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:07.937137 3643 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk I0521 19:40:07.937142 3643 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-pgqr7] creating logical port openshift-insights_insights-operator-6dbb575d6c-pgqr7 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:07.937316 3643 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-pgqr7 I0521 19:40:07.955675 3643 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk] pod update took 18.560595ms I0521 19:40:07.955701 3643 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk] addLogicalPort annotation time took 18.588866ms I0521 19:40:07.956811 3643 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk] addLogicalPort took 19.810341ms, libovsdb time 823.725µs I0521 19:40:07.971630 3643 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-pgqr7] pod update took 34.345343ms I0521 19:40:07.971652 3643 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-pgqr7] addLogicalPort annotation time took 34.372184ms I0521 19:40:07.972640 3643 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-pgqr7] addLogicalPort took 35.511642ms, libovsdb time 649.902µs I0521 19:40:08.025021 3643 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-2jcz6 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:08.025124 3643 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6 I0521 19:40:08.025175 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-g244s for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:08.025251 3643 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s I0521 19:40:08.044645 3643 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6] pod update took 19.552908ms I0521 19:40:08.044666 3643 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6] addLogicalPort annotation time took 19.578327ms I0521 19:40:08.045697 3643 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6] addLogicalPort took 20.693219ms, libovsdb time 748.897µs I0521 19:40:08.048044 3643 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s] pod update took 22.8106ms I0521 19:40:08.048062 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s] addLogicalPort annotation time took 22.832533ms I0521 19:40:08.048971 3643 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s] addLogicalPort took 23.803791ms, libovsdb time 656.636µs I0521 19:40:08.157621 3643 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp 2eb5a51cac67b2e3be04059690ad2f8fa86c8c948f3ab9d4681baa4bc4bb0e36 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp 2eb5a51cac67b2e3be04059690ad2f8fa86c8c948f3ab9d4681baa4bc4bb0e36 network default NAD default NAD key ] I0521 19:40:08.168688 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-9xbfp, hostIfaceName: 2eb5a51cac67b2e, network: default, NAD default, SandboxID: "2eb5a51cac67b2e3be04059690ad2f8fa86c8c948f3ab9d4681baa4bc4bb0e36", PCI device ID: , UID: "816b0c67-53de-42db-9dec-0ff846ee252c", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0521 19:40:08.215798 3643 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp 2eb5a51cac67b2e3be04059690ad2f8fa86c8c948f3ab9d4681baa4bc4bb0e36 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9xbfp 2eb5a51cac67b2e3be04059690ad2f8fa86c8c948f3ab9d4681baa4bc4bb0e36 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:c4:95:22:43:a7\",\"name\":\"2eb5a51cac67b2e\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/00f27f4a-1b8d-4198-9eac-1e9296e6737c\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:08.290620 3643 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk facad46a8977713006b19276621e80b9c751d4b57c424457416ed9b85105d88b network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk facad46a8977713006b19276621e80b9c751d4b57c424457416ed9b85105d88b network default NAD default NAD key ] I0521 19:40:08.291634 3643 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-pgqr7 10f13561fad4d04d4e678614175bc990f9b09aa9731ed6eb2d8c12e62ae63d1a network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-pgqr7 10f13561fad4d04d4e678614175bc990f9b09aa9731ed6eb2d8c12e62ae63d1a network default NAD default NAD key ] I0521 19:40:08.303894 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-pgqr7, hostIfaceName: 10f13561fad4d04, network: default, NAD default, SandboxID: "10f13561fad4d04d4e678614175bc990f9b09aa9731ed6eb2d8c12e62ae63d1a", PCI device ID: , UID: "4a913d19-d192-479a-bf32-b2cf0eb723a7", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0521 19:40:08.304598 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-zmprk, hostIfaceName: facad46a8977713, network: default, NAD default, SandboxID: "facad46a8977713006b19276621e80b9c751d4b57c424457416ed9b85105d88b", PCI device ID: , UID: "dadaaa7f-2c93-4928-a094-f674c379d109", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0521 19:40:08.349172 3643 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-pgqr7 10f13561fad4d04d4e678614175bc990f9b09aa9731ed6eb2d8c12e62ae63d1a network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-pgqr7 10f13561fad4d04d4e678614175bc990f9b09aa9731ed6eb2d8c12e62ae63d1a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:05:52:29:78:cd\",\"name\":\"10f13561fad4d04\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc3bb898-dffe-44d9-8e0b-ce4e58d5ccd2\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:08.363930 3643 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk facad46a8977713006b19276621e80b9c751d4b57c424457416ed9b85105d88b network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-zmprk facad46a8977713006b19276621e80b9c751d4b57c424457416ed9b85105d88b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:62:66:df:5d:78\",\"name\":\"facad46a8977713\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba98b1a6-d525-4b79-a2e3-91bd49921788\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:11.143006 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0521 19:40:11.501844 3643 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0521 19:40:11.503236 3643 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.369581ms I0521 19:40:11.528312 3643 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0521 19:40:13.251018 3643 namespace.go:98] [openshift-service-ca] adding namespace I0521 19:40:13.251840 3643 namespace.go:102] [openshift-service-ca] adding namespace took 803.345µs I0521 19:40:13.267374 3643 namespace.go:142] [openshift-service-ca] updating namespace W0521 19:40:22.020330 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0521 19:40:23.981359 3643 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6 032dd041a40885c6c6d679e29224d46099da9ff5779e7fc5e600d21461578fc8 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6 032dd041a40885c6c6d679e29224d46099da9ff5779e7fc5e600d21461578fc8 network default NAD default NAD key ] I0521 19:40:23.991205 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-2jcz6, hostIfaceName: 032dd041a40885c, network: default, NAD default, SandboxID: "032dd041a40885c6c6d679e29224d46099da9ff5779e7fc5e600d21461578fc8", PCI device ID: , UID: "4abc117c-b702-4cbc-9eef-371c8eaf07d3", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0521 19:40:24.040848 3643 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6 032dd041a40885c6c6d679e29224d46099da9ff5779e7fc5e600d21461578fc8 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-2jcz6 032dd041a40885c6c6d679e29224d46099da9ff5779e7fc5e600d21461578fc8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:cb:c0:ea:70:46\",\"name\":\"032dd041a40885c\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffcfade6-1da3-48c2-88f3-3c32e95b8fcf\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:29.735160 3643 cni.go:452] [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key ] I0521 19:40:29.735191 3643 cni.go:452] [openshift-dns/dns-default-lfrcc 22172e85db9c65d753ed3bbfa57a6ae77a985930742bacf53e9b7005f727c864 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-lfrcc 22172e85db9c65d753ed3bbfa57a6ae77a985930742bacf53e9b7005f727c864 network default NAD default NAD key ] I0521 19:40:29.745290 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-lfrcc, hostIfaceName: 22172e85db9c65d, network: default, NAD default, SandboxID: "22172e85db9c65d753ed3bbfa57a6ae77a985930742bacf53e9b7005f727c864", PCI device ID: , UID: "fa21d3c9-59cd-4830-87d3-cbb545b18249", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0521 19:40:29.745468 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6778768bb7-g9rlf, hostIfaceName: 0eff374bf8bbc56, network: default, NAD default, SandboxID: "0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6", PCI device ID: , UID: "e357dbf0-f66c-4de5-bce4-793cec7effcc", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0521 19:40:29.793196 3643 cni.go:473] [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:e4:ff:94:e1:9a\",\"name\":\"0eff374bf8bbc56\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4967df41-1119-4eb1-b224-bfeced8724a4\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:29.808393 3643 cni.go:473] [openshift-dns/dns-default-lfrcc 22172e85db9c65d753ed3bbfa57a6ae77a985930742bacf53e9b7005f727c864 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-lfrcc 22172e85db9c65d753ed3bbfa57a6ae77a985930742bacf53e9b7005f727c864 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:7b:fc:3b:3a:44\",\"name\":\"22172e85db9c65d\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/480521db-166c-4f31-ade3-62d079f9b149\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:39.187057 3643 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:40:39.211377 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:39.211403 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:39.211413 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:39.304824 3643 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:40:39.320450 3643 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0521 19:40:39.323571 3643 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-qb7bg] creating logical port openshift-insights_insights-runtime-extractor-qb7bg for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:39.323661 3643 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-qb7bg I0521 19:40:39.328371 3643 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-9bk8l] creating logical port openshift-console_downloads-5c54f4944f-9bk8l for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:39.328463 3643 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-9bk8l I0521 19:40:39.364827 3643 pod.go:62] [openshift-insights/insights-runtime-extractor-qb7bg] pod update took 41.190919ms I0521 19:40:39.364849 3643 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-qb7bg] addLogicalPort annotation time took 41.217905ms I0521 19:40:39.364942 3643 pod.go:62] [openshift-console/downloads-5c54f4944f-9bk8l] pod update took 36.508024ms I0521 19:40:39.364967 3643 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-9bk8l] addLogicalPort annotation time took 36.53449ms I0521 19:40:39.365993 3643 pods.go:271] [openshift-insights/insights-runtime-extractor-qb7bg] addLogicalPort took 42.438137ms, libovsdb time 790.867µs I0521 19:40:39.366276 3643 pods.go:271] [openshift-console/downloads-5c54f4944f-9bk8l] addLogicalPort took 37.918848ms, libovsdb time 1.008314ms I0521 19:40:39.709021 3643 cni.go:452] [openshift-insights/insights-runtime-extractor-qb7bg 20efbedecc418e795b551abb0ec9d7472225c6c7fdb0aa618ec4463cc134cf75 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-qb7bg 20efbedecc418e795b551abb0ec9d7472225c6c7fdb0aa618ec4463cc134cf75 network default NAD default NAD key ] I0521 19:40:39.718551 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-qb7bg, hostIfaceName: 20efbedecc418e7, network: default, NAD default, SandboxID: "20efbedecc418e795b551abb0ec9d7472225c6c7fdb0aa618ec4463cc134cf75", PCI device ID: , UID: "f0265019-bf96-4d78-b4ab-6a8deaac324b", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0521 19:40:39.733880 3643 cni.go:452] [openshift-console/downloads-5c54f4944f-9bk8l 840c58d2b886ad28ae950686f2e72dbfeedd23d1cc9102681267ffd6f9ec9e95 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-9bk8l 840c58d2b886ad28ae950686f2e72dbfeedd23d1cc9102681267ffd6f9ec9e95 network default NAD default NAD key ] I0521 19:40:39.743428 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-9bk8l, hostIfaceName: 840c58d2b886ad2, network: default, NAD default, SandboxID: "840c58d2b886ad28ae950686f2e72dbfeedd23d1cc9102681267ffd6f9ec9e95", PCI device ID: , UID: "56968800-bba1-4b6e-8da1-7405f8cfcb9d", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0521 19:40:39.772896 3643 cni.go:473] [openshift-insights/insights-runtime-extractor-qb7bg 20efbedecc418e795b551abb0ec9d7472225c6c7fdb0aa618ec4463cc134cf75 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-qb7bg 20efbedecc418e795b551abb0ec9d7472225c6c7fdb0aa618ec4463cc134cf75 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:f3:6a:50:7b:e5\",\"name\":\"20efbedecc418e7\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b774adf-706a-43b1-afcb-a6e239a7ba8a\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:39.790644 3643 cni.go:473] [openshift-console/downloads-5c54f4944f-9bk8l 840c58d2b886ad28ae950686f2e72dbfeedd23d1cc9102681267ffd6f9ec9e95 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-9bk8l 840c58d2b886ad28ae950686f2e72dbfeedd23d1cc9102681267ffd6f9ec9e95 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:71:19:2c:3c:68\",\"name\":\"840c58d2b886ad2\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dda68c3f-88c7-4263-819c-5047800f61ea\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:39.981289 3643 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-b577n 2675a7113bb51ebfd81e228caa6c94dc1236c835a0af42ed47ecda1075687f87 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-b577n 2675a7113bb51ebfd81e228caa6c94dc1236c835a0af42ed47ecda1075687f87 network default NAD default NAD key ] I0521 19:40:39.990441 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-b577n, hostIfaceName: 2675a7113bb51eb, network: default, NAD default, SandboxID: "2675a7113bb51ebfd81e228caa6c94dc1236c835a0af42ed47ecda1075687f87", PCI device ID: , UID: "988a9f4d-0632-4182-8196-24ac1611b1a6", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0521 19:40:40.035662 3643 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-b577n 2675a7113bb51ebfd81e228caa6c94dc1236c835a0af42ed47ecda1075687f87 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-b577n 2675a7113bb51ebfd81e228caa6c94dc1236c835a0af42ed47ecda1075687f87 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:b1:bf:9a:94:b3\",\"name\":\"2675a7113bb51eb\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e13443b-14b6-414a-a810-fe382e7a784a\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:40.198384 3643 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s 16a96a9e4292bcf3f1e82791d4777706db6b70d11097f7713b0195a9c6315e01 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s 16a96a9e4292bcf3f1e82791d4777706db6b70d11097f7713b0195a9c6315e01 network default NAD default NAD key ] I0521 19:40:40.208766 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-g244s, hostIfaceName: 16a96a9e4292bcf, network: default, NAD default, SandboxID: "16a96a9e4292bcf3f1e82791d4777706db6b70d11097f7713b0195a9c6315e01", PCI device ID: , UID: "fbcbd6ce-355b-4d63-9e57-f8eef70bcd63", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0521 19:40:40.256351 3643 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s 16a96a9e4292bcf3f1e82791d4777706db6b70d11097f7713b0195a9c6315e01 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-g244s 16a96a9e4292bcf3f1e82791d4777706db6b70d11097f7713b0195a9c6315e01 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:44:a8:51:4b:f7\",\"name\":\"16a96a9e4292bcf\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6aeabf3b-9547-41ec-947b-7aaa67484aa0\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:40.275522 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0521 19:40:40.275540 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0521 19:40:40.275653 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.275669 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.275675 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.938014 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0521 19:40:40.938039 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0521 19:40:40.939250 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.939269 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.939277 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.966674 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0521 19:40:40.966692 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0521 19:40:40.969498 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.969515 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:40.969525 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0521 19:40:41.218575 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:41.218596 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:41.218602 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:41.218614 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:41.218619 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:41.218625 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:42.259606 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:42.259693 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0521 19:40:42.259709 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0521 19:40:42.874586 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-7d58z for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:42.874787 3643 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z I0521 19:40:42.891262 3643 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z] pod update took 16.516171ms I0521 19:40:42.891287 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z] addLogicalPort annotation time took 16.547198ms I0521 19:40:42.892700 3643 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z] addLogicalPort took 18.13549ms, libovsdb time 1.063858ms I0521 19:40:43.042414 3643 base_network_controller_pods.go:487] [default/openshift-console/console-644c565d4b-vf7tn] creating logical port openshift-console_console-644c565d4b-vf7tn for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:43.042593 3643 kube.go:256] Updating pod openshift-console/console-644c565d4b-vf7tn I0521 19:40:43.063510 3643 pod.go:62] [openshift-console/console-644c565d4b-vf7tn] pod update took 20.941359ms I0521 19:40:43.063532 3643 base_network_controller_pods.go:951] [default/openshift-console/console-644c565d4b-vf7tn] addLogicalPort annotation time took 20.971086ms I0521 19:40:43.064766 3643 pods.go:271] [openshift-console/console-644c565d4b-vf7tn] addLogicalPort took 22.375316ms, libovsdb time 885.934µs I0521 19:40:43.408761 3643 cni.go:452] [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key ] I0521 19:40:43.420378 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-644c565d4b-vf7tn, hostIfaceName: 9044a3da0783fdc, network: default, NAD default, SandboxID: "9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483", PCI device ID: , UID: "d486c791-fe48-4248-b58c-4a013e292d4b", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0521 19:40:43.473005 3643 cni.go:473] [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:c2:ce:0c:d4:8c\",\"name\":\"9044a3da0783fdc\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0f824611-9513-46e3-982a-9ba3a51c58fa\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:43.835268 3643 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z fd78ec3f675bb5e24118901f6605679f0a80124efa278a581baa7b880c438d68 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z fd78ec3f675bb5e24118901f6605679f0a80124efa278a581baa7b880c438d68 network default NAD default NAD key ] I0521 19:40:43.845843 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-7d58z, hostIfaceName: fd78ec3f675bb5e, network: default, NAD default, SandboxID: "fd78ec3f675bb5e24118901f6605679f0a80124efa278a581baa7b880c438d68", PCI device ID: , UID: "8c2bb04e-bf4b-4f92-95fd-bba45280ac51", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0521 19:40:43.900574 3643 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z fd78ec3f675bb5e24118901f6605679f0a80124efa278a581baa7b880c438d68 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-7d58z fd78ec3f675bb5e24118901f6605679f0a80124efa278a581baa7b880c438d68 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:0f:d6:03:3e:be\",\"name\":\"fd78ec3f675bb5e\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eeacd56c-3060-43c0-8bdb-c5d0be7fa4fd\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:45.271097 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0521 19:40:45.327390 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0521 19:40:45.995233 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-xghp5] creating logical port openshift-monitoring_prometheus-operator-76b589d668-xghp5 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:45.995353 3643 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-xghp5 I0521 19:40:46.011231 3643 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-xghp5] pod update took 15.906354ms I0521 19:40:46.011258 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-xghp5] addLogicalPort annotation time took 15.938812ms I0521 19:40:46.012691 3643 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-xghp5] addLogicalPort took 17.474313ms, libovsdb time 1.082714ms I0521 19:40:46.961866 3643 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-xghp5 3607f914685668794880c9ac7cc83d7183e260864d96877e50b2daa4130cbcff network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-xghp5 3607f914685668794880c9ac7cc83d7183e260864d96877e50b2daa4130cbcff network default NAD default NAD key ] I0521 19:40:46.973997 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-xghp5, hostIfaceName: 3607f9146856687, network: default, NAD default, SandboxID: "3607f914685668794880c9ac7cc83d7183e260864d96877e50b2daa4130cbcff", PCI device ID: , UID: "7a2471fc-4722-45f0-83ba-cc32cdbffd9c", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0521 19:40:47.023331 3643 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-xghp5 3607f914685668794880c9ac7cc83d7183e260864d96877e50b2daa4130cbcff network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-xghp5 3607f914685668794880c9ac7cc83d7183e260864d96877e50b2daa4130cbcff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:cd:13:80:36:18\",\"name\":\"3607f9146856687\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cbe7506c-876c-4d0d-af5c-2f7bf41dbf2b\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:47.307598 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0521 19:40:47.985766 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0521 19:40:48.289121 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0521 19:40:49.308093 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0521 19:40:51.910517 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-c925h] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-c925h for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:51.910616 3643 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-c925h I0521 19:40:51.957984 3643 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-c925h I0521 19:40:52.005358 3643 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-c925h] pod update took 94.77367ms I0521 19:40:52.005386 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-c925h] addLogicalPort annotation time took 94.807089ms I0521 19:40:52.006653 3643 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-c925h] addLogicalPort took 96.151219ms, libovsdb time 940.746µs W0521 19:40:54.097476 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:40:54.097511 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0521 19:40:54.814774 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-b548bc4cd-g2l45] creating logical port openshift-monitoring_thanos-querier-b548bc4cd-g2l45 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:54.814957 3643 kube.go:256] Updating pod openshift-monitoring/thanos-querier-b548bc4cd-g2l45 I0521 19:40:54.843674 3643 pod.go:62] [openshift-monitoring/thanos-querier-b548bc4cd-g2l45] pod update took 28.799664ms I0521 19:40:54.843704 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-b548bc4cd-g2l45] addLogicalPort annotation time took 28.834325ms I0521 19:40:54.845005 3643 pods.go:271] [openshift-monitoring/thanos-querier-b548bc4cd-g2l45] addLogicalPort took 30.24878ms, libovsdb time 902.913µs I0521 19:40:55.188841 3643 cni.go:452] [openshift-monitoring/thanos-querier-b548bc4cd-g2l45 5e248f2559db46b69411a42e1fd997cfb417232c74bf6351f3f1ff0d0c9f4877 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-b548bc4cd-g2l45 5e248f2559db46b69411a42e1fd997cfb417232c74bf6351f3f1ff0d0c9f4877 network default NAD default NAD key ] I0521 19:40:55.190707 3643 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-c925h 195098b0d34414afd65d11e04ceb67a41cff95d7729a91005985ae17331bfa9e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-c925h 195098b0d34414afd65d11e04ceb67a41cff95d7729a91005985ae17331bfa9e network default NAD default NAD key ] I0521 19:40:55.201451 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-b548bc4cd-g2l45, hostIfaceName: 5e248f2559db46b, network: default, NAD default, SandboxID: "5e248f2559db46b69411a42e1fd997cfb417232c74bf6351f3f1ff0d0c9f4877", PCI device ID: , UID: "300e0696-5b12-405f-a13a-da211bad0aa2", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0521 19:40:55.201648 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-c925h, hostIfaceName: 195098b0d34414a, network: default, NAD default, SandboxID: "195098b0d34414afd65d11e04ceb67a41cff95d7729a91005985ae17331bfa9e", PCI device ID: , UID: "3ac5b12d-6d53-4085-80c7-70ea9e72e351", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0521 19:40:55.253893 3643 cni.go:473] [openshift-monitoring/thanos-querier-b548bc4cd-g2l45 5e248f2559db46b69411a42e1fd997cfb417232c74bf6351f3f1ff0d0c9f4877 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-b548bc4cd-g2l45 5e248f2559db46b69411a42e1fd997cfb417232c74bf6351f3f1ff0d0c9f4877 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:44:1f:89:7e:2e\",\"name\":\"5e248f2559db46b\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70e2ef8e-23f3-45c5-a5b7-5bad8d5f3499\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:55.273011 3643 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-c925h 195098b0d34414afd65d11e04ceb67a41cff95d7729a91005985ae17331bfa9e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-c925h 195098b0d34414afd65d11e04ceb67a41cff95d7729a91005985ae17331bfa9e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:97:b1:18:7c:b8\",\"name\":\"195098b0d34414a\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9df7d10d-263d-43a5-a94a-ffbce564d0aa\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:55.319028 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0521 19:40:55.349653 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0521 19:40:56.370585 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:40:56.370615 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:40:56.444721 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0521 19:40:56.483154 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:40:56.483192 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0521 19:40:56.675190 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-d7764bc7b-kqpdw] creating logical port openshift-monitoring_metrics-server-d7764bc7b-kqpdw for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:56.675318 3643 kube.go:256] Updating pod openshift-monitoring/metrics-server-d7764bc7b-kqpdw I0521 19:40:56.722222 3643 kube.go:256] Updating pod openshift-monitoring/metrics-server-d7764bc7b-kqpdw I0521 19:40:56.755615 3643 pod.go:62] [openshift-monitoring/metrics-server-d7764bc7b-kqpdw] pod update took 80.322506ms I0521 19:40:56.755648 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-d7764bc7b-kqpdw] addLogicalPort annotation time took 80.360246ms I0521 19:40:56.757118 3643 pods.go:271] [openshift-monitoring/metrics-server-d7764bc7b-kqpdw] addLogicalPort took 81.94682ms, libovsdb time 1.051101ms I0521 19:40:57.331958 3643 cni.go:452] [openshift-monitoring/metrics-server-d7764bc7b-kqpdw 2047a435597a132041cdd1b813dda8ff1e3dd340ca3172a8078f5f18061c8adc network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-d7764bc7b-kqpdw 2047a435597a132041cdd1b813dda8ff1e3dd340ca3172a8078f5f18061c8adc network default NAD default NAD key ] I0521 19:40:57.343280 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-d7764bc7b-kqpdw, hostIfaceName: 2047a435597a132, network: default, NAD default, SandboxID: "2047a435597a132041cdd1b813dda8ff1e3dd340ca3172a8078f5f18061c8adc", PCI device ID: , UID: "292fffa7-944f-48a8-b33c-e5817fbd021f", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0521 19:40:57.401127 3643 cni.go:473] [openshift-monitoring/metrics-server-d7764bc7b-kqpdw 2047a435597a132041cdd1b813dda8ff1e3dd340ca3172a8078f5f18061c8adc network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-d7764bc7b-kqpdw 2047a435597a132041cdd1b813dda8ff1e3dd340ca3172a8078f5f18061c8adc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:8c:8b:8e:f7:0c\",\"name\":\"2047a435597a132\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d86f87ce-d402-4b23-a9bc-638cf38ad897\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:40:57.407287 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:40:57.407303 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:40:57.407308 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:40:58.051539 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:40:58.051557 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:40:58.051564 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0521 19:40:58.156969 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:40:58.157613 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:40:58.157655 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:40:58.157668 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0521 19:40:58.157682 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:40:58.157690 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0521 19:40:58.349327 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:40:58.349444 3643 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0521 19:40:58.377793 3643 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 28.390098ms I0521 19:40:58.377820 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 28.425295ms I0521 19:40:58.379213 3643 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 29.906579ms, libovsdb time 985.954µs I0521 19:40:58.950312 3643 cni.go:452] [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key ] I0521 19:40:58.962646 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 37169407919ffb4, network: default, NAD default, SandboxID: "37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190", PCI device ID: , UID: "37fbf48f-1d23-4c8b-bc76-924450c188e2", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0521 19:40:59.026943 3643 cni.go:473] [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:b1:d7:47:9c:e1\",\"name\":\"37169407919ffb4\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2858bf76-d45d-40f5-9319-66d2fbfe1bbe\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:40:59.188861 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:40:59.189187 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:40:59.189248 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:40:59.189263 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d55n7t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:40:59.353859 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0521 19:40:59.377593 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0521 19:40:59.378480 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0521 19:40:59.407621 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:40:59.407642 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:40:59.407649 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:40:59.407654 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:40:59.839496 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-c6d8d7bf8-mbqk4 IP address from the namespace address-set, err: pod openshift-console/console-c6d8d7bf8-mbqk4: no pod IPs found W0521 19:41:00.050398 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0521 19:41:00.131694 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:00.131719 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:00.160251 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:00.160269 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:00.160274 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:00.160278 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:00.409736 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:00.409759 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0521 19:41:03.779175 3643 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0521 19:41:03.791893 3643 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0521 19:41:04.433933 3643 cni.go:452] [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key ] I0521 19:41:04.521195 3643 cni.go:473] [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-6778768bb7-g9rlf 0eff374bf8bbc560a3ad31eb6d0be4521c4b97eb80d8c9cf5d09ab0584a5f2e6 network default NAD default NAD key default], result "{}", err I0521 19:41:04.571082 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6778768bb7-g9rlf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:04.571151 3643 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6778768bb7-g9rlf I0521 19:41:04.572635 3643 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6778768bb7-g9rlf, ips: 10.130.0.6 I0521 19:41:04.572666 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6778768bb7-g9rlf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:04.582484 3643 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6db74d78b4-d2bwb] creating logical port openshift-image-registry_image-registry-6db74d78b4-d2bwb for pod on switch ip-10-0-2-167.ec2.internal I0521 19:41:04.582591 3643 kube.go:256] Updating pod openshift-image-registry/image-registry-6db74d78b4-d2bwb I0521 19:41:04.607364 3643 pod.go:62] [openshift-image-registry/image-registry-6db74d78b4-d2bwb] pod update took 24.798628ms I0521 19:41:04.607390 3643 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6db74d78b4-d2bwb] addLogicalPort annotation time took 24.828588ms I0521 19:41:04.608628 3643 pods.go:271] [openshift-image-registry/image-registry-6db74d78b4-d2bwb] addLogicalPort took 26.157158ms, libovsdb time 850.587µs I0521 19:41:04.622199 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-f7f9cdd6d-2b776 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:04.622623 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-f7f9cdd6d-2b776 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:04.939948 3643 cni.go:452] [openshift-image-registry/image-registry-6db74d78b4-d2bwb 5e0f5ca1db2d1c3d5f78ce11797c7ba0a8c46c808e1500fa5aa89c10e8515266 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6db74d78b4-d2bwb 5e0f5ca1db2d1c3d5f78ce11797c7ba0a8c46c808e1500fa5aa89c10e8515266 network default NAD default NAD key ] I0521 19:41:04.949188 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6db74d78b4-d2bwb, hostIfaceName: 5e0f5ca1db2d1c3, network: default, NAD default, SandboxID: "5e0f5ca1db2d1c3d5f78ce11797c7ba0a8c46c808e1500fa5aa89c10e8515266", PCI device ID: , UID: "5c58aa47-02c3-4864-a6a7-8e55adf55f88", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0521 19:41:04.998031 3643 cni.go:473] [openshift-image-registry/image-registry-6db74d78b4-d2bwb 5e0f5ca1db2d1c3d5f78ce11797c7ba0a8c46c808e1500fa5aa89c10e8515266 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6db74d78b4-d2bwb 5e0f5ca1db2d1c3d5f78ce11797c7ba0a8c46c808e1500fa5aa89c10e8515266 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:72:6b:ca:9d:94\",\"name\":\"5e0f5ca1db2d1c3\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/63274307-8e93-41e8-b72b-4cd896a27d37\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:41:05.077383 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-f7f9cdd6d-2b776 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:05.077405 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-f7f9cdd6d-2b776 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:05.086492 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-f7f9cdd6d-2b776 I0521 19:41:05.086511 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-f7f9cdd6d-2b776 W0521 19:41:05.399666 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:05.399690 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:05.399696 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:41:05.399701 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0521 19:41:05.440038 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6778768bb7-g9rlf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:05.440067 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6778768bb7-g9rlf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:05.445923 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6778768bb7-g9rlf I0521 19:41:05.445945 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6778768bb7-g9rlf W0521 19:41:07.066263 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-5dbdbc9584-jq8vp IP address from the namespace address-set, err: pod openshift-console/console-5dbdbc9584-jq8vp: no pod IPs found W0521 19:41:07.076230 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-5dbdbc9584-qz76f IP address from the namespace address-set, err: pod openshift-console/console-5dbdbc9584-qz76f: no pod IPs found I0521 19:41:12.460173 3643 cni.go:452] [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key ] I0521 19:41:12.547558 3643 cni.go:473] [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-644c565d4b-vf7tn 9044a3da0783fdc72b6d9da4aae1fdd47106fdf93d3ae3a0a9b6b6d1915dd483 network default NAD default NAD key default], result "{}", err I0521 19:41:12.614537 3643 obj_retry.go:498] Detected object openshift-console/console-644c565d4b-vf7tn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:12.614572 3643 pods.go:176] Deleting pod: openshift-console/console-644c565d4b-vf7tn I0521 19:41:12.615989 3643 pods.go:236] Attempting to release IPs for pod: openshift-console/console-644c565d4b-vf7tn, ips: 10.130.0.18 I0521 19:41:12.616018 3643 obj_retry.go:498] Detected object openshift-console/console-644c565d4b-vf7tn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:12.623820 3643 base_network_controller_pods.go:487] [default/openshift-console/console-648459754-pfb9s] creating logical port openshift-console_console-648459754-pfb9s for pod on switch ip-10-0-2-167.ec2.internal I0521 19:41:12.623959 3643 kube.go:256] Updating pod openshift-console/console-648459754-pfb9s I0521 19:41:12.642443 3643 pod.go:62] [openshift-console/console-648459754-pfb9s] pod update took 18.510429ms I0521 19:41:12.642470 3643 base_network_controller_pods.go:951] [default/openshift-console/console-648459754-pfb9s] addLogicalPort annotation time took 18.541996ms I0521 19:41:12.643815 3643 pods.go:271] [openshift-console/console-648459754-pfb9s] addLogicalPort took 20.009151ms, libovsdb time 988.038µs I0521 19:41:13.020929 3643 cni.go:452] [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key ] I0521 19:41:13.030858 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-648459754-pfb9s, hostIfaceName: 75c382c81db79d8, network: default, NAD default, SandboxID: "75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79", PCI device ID: , UID: "b05ae7ed-9d92-4e32-a5a9-e8c2eba693d8", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0521 19:41:13.081520 3643 cni.go:473] [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:56:fd:ae:b8:0a\",\"name\":\"75c382c81db79d8\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dfcbbe51-2985-405e-bce6-44c0866a8a17\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:41:13.470309 3643 obj_retry.go:492] Detected object openshift-console/console-644c565d4b-vf7tn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:13.470328 3643 obj_retry.go:492] Detected object openshift-console/console-644c565d4b-vf7tn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:13.472157 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-644c565d4b-vf7tn I0521 19:41:13.472183 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-644c565d4b-vf7tn W0521 19:41:20.366479 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0521 19:41:20.366505 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0521 19:41:20.398212 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0521 19:41:21.764752 3643 obj_retry.go:498] Detected object openshift-console/console-644c565d4b-kfjsk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:21.765375 3643 obj_retry.go:498] Detected object openshift-console/console-644c565d4b-kfjsk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:22.154842 3643 obj_retry.go:492] Detected object openshift-console/console-644c565d4b-kfjsk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:22.154869 3643 obj_retry.go:492] Detected object openshift-console/console-644c565d4b-kfjsk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:22.160530 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-644c565d4b-kfjsk I0521 19:41:22.160556 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-644c565d4b-kfjsk W0521 19:41:23.518895 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:41:26.430887 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:26.470677 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0521 19:41:31.534051 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0521 19:41:31.534118 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0521 19:41:31.534162 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0521 19:41:32.549135 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0521 19:41:37.563262 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0521 19:41:37.563304 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0521 19:41:37.563351 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0521 19:41:38.572294 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0521 19:41:48.879583 3643 obj_retry.go:498] Detected object openshift-console/console-c6d8d7bf8-2bx7j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:48.880150 3643 obj_retry.go:498] Detected object openshift-console/console-c6d8d7bf8-2bx7j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:49.549339 3643 obj_retry.go:492] Detected object openshift-console/console-c6d8d7bf8-2bx7j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:49.549363 3643 obj_retry.go:492] Detected object openshift-console/console-c6d8d7bf8-2bx7j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:49.556879 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c6d8d7bf8-2bx7j I0521 19:41:49.556919 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c6d8d7bf8-2bx7j I0521 19:41:52.781804 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-f7f9cdd6d-flm8k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:52.782411 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-f7f9cdd6d-flm8k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:53.570656 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-f7f9cdd6d-flm8k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:53.570687 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-f7f9cdd6d-flm8k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:53.577027 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-f7f9cdd6d-flm8k I0521 19:41:53.577052 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-f7f9cdd6d-flm8k I0521 19:41:57.319432 3643 obj_retry.go:498] Detected object openshift-ingress/router-default-679c7cf987-rnwjc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:57.320034 3643 obj_retry.go:498] Detected object openshift-ingress/router-default-679c7cf987-rnwjc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:41:58.296923 3643 obj_retry.go:492] Detected object openshift-ingress/router-default-679c7cf987-rnwjc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:58.296948 3643 obj_retry.go:492] Detected object openshift-ingress/router-default-679c7cf987-rnwjc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:41:58.303283 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-679c7cf987-rnwjc I0521 19:41:58.303305 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-679c7cf987-rnwjc W0521 19:41:59.349622 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:59.349645 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:59.349658 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:59.349663 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:59.693803 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:41:59.693824 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0521 19:42:03.810802 3643 cni.go:452] [openshift-multus/network-metrics-daemon-sc58z f54270d5254fd0b5cb4bf75a72ae737c1a9651475c61ad827aa173fb1ddc55fa network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-sc58z f54270d5254fd0b5cb4bf75a72ae737c1a9651475c61ad827aa173fb1ddc55fa network default NAD default NAD key ] I0521 19:42:03.820997 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-sc58z, hostIfaceName: f54270d5254fd0b, network: default, NAD default, SandboxID: "f54270d5254fd0b5cb4bf75a72ae737c1a9651475c61ad827aa173fb1ddc55fa", PCI device ID: , UID: "0ef9d357-6938-4d76-a4a7-8f05654b5df0", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0521 19:42:03.869678 3643 cni.go:473] [openshift-multus/network-metrics-daemon-sc58z f54270d5254fd0b5cb4bf75a72ae737c1a9651475c61ad827aa173fb1ddc55fa network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-sc58z f54270d5254fd0b5cb4bf75a72ae737c1a9651475c61ad827aa173fb1ddc55fa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:ce:2d:e2:21:b4\",\"name\":\"f54270d5254fd0b\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ca67c924-026b-44aa-a28b-8149cd3810c6\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:42:13.959522 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:13.960059 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:14.375565 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:14.375588 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:14.383787 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0521 19:42:14.383816 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0521 19:42:18.237217 3643 cni.go:452] [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key ] I0521 19:42:18.338924 3643 cni.go:473] [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 37169407919ffb47631a597f1ac88d4b64e29a62c4c13b8045cc6292cc811190 network default NAD default NAD key default], result "{}", err I0521 19:42:18.403688 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:18.403721 3643 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0521 19:42:18.405109 3643 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.130.0.23 I0521 19:42:18.405137 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:18.745624 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:18.745653 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:18.751839 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0521 19:42:18.751866 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0521 19:42:18.791731 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:42:18.791876 3643 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0521 19:42:18.817592 3643 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 25.793637ms I0521 19:42:18.817620 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 25.826977ms I0521 19:42:18.818839 3643 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 27.123938ms, libovsdb time 837.976µs I0521 19:42:18.832586 3643 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0521 19:42:19.148937 3643 cni.go:452] [openshift-monitoring/prometheus-k8s-1 6d88b7d0cb3fd59120c1f827f91cdf270d7d035988b7618b817488dfb2cd3f43 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 6d88b7d0cb3fd59120c1f827f91cdf270d7d035988b7618b817488dfb2cd3f43 network default NAD default NAD key ] I0521 19:42:19.158637 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 6d88b7d0cb3fd59, network: default, NAD default, SandboxID: "6d88b7d0cb3fd59120c1f827f91cdf270d7d035988b7618b817488dfb2cd3f43", PCI device ID: , UID: "1f27da90-3cd3-4da6-a5b7-77438a09631f", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0521 19:42:19.209791 3643 cni.go:473] [openshift-monitoring/prometheus-k8s-1 6d88b7d0cb3fd59120c1f827f91cdf270d7d035988b7618b817488dfb2cd3f43 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 6d88b7d0cb3fd59120c1f827f91cdf270d7d035988b7618b817488dfb2cd3f43 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:1d:ab:fd:8f:8a\",\"name\":\"6d88b7d0cb3fd59\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/588ccb6f-3051-4aa0-8255-dbc64530c9ee\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:42:38.800263 3643 obj_retry.go:498] Detected object openshift-console/console-648459754-fm2tq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:38.804378 3643 obj_retry.go:498] Detected object openshift-console/console-648459754-fm2tq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:39.508300 3643 obj_retry.go:492] Detected object openshift-console/console-648459754-fm2tq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:39.508322 3643 obj_retry.go:492] Detected object openshift-console/console-648459754-fm2tq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:39.515402 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-648459754-fm2tq I0521 19:42:39.515424 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-648459754-fm2tq I0521 19:42:46.167771 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:46.168369 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:46.780126 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:46.780170 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:46.790335 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0521 19:42:46.790367 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0521 19:42:49.901830 3643 cni.go:452] [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key ] I0521 19:42:49.987342 3643 cni.go:473] [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-648459754-pfb9s 75c382c81db79d8f509ee04b620b2a6e28b23e70406423f6bcb460a54a64fc79 network default NAD default NAD key default], result "{}", err I0521 19:42:50.041477 3643 obj_retry.go:498] Detected object openshift-console/console-648459754-pfb9s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:50.041515 3643 pods.go:176] Deleting pod: openshift-console/console-648459754-pfb9s I0521 19:42:50.042725 3643 pods.go:236] Attempting to release IPs for pod: openshift-console/console-648459754-pfb9s, ips: 10.130.0.25 I0521 19:42:50.042756 3643 obj_retry.go:498] Detected object openshift-console/console-648459754-pfb9s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:50.887178 3643 obj_retry.go:492] Detected object openshift-console/console-648459754-pfb9s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:50.887209 3643 obj_retry.go:492] Detected object openshift-console/console-648459754-pfb9s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:50.891187 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-648459754-pfb9s I0521 19:42:50.891222 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-648459754-pfb9s I0521 19:43:03.972979 3643 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-mflsn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:03.973491 3643 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-mflsn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:04.877639 3643 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-mflsn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:04.877665 3643 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-mflsn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:16.799650 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=586 I0521 19:43:16.886711 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=5 W0521 19:43:20.047821 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:43:20.047852 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0521 19:43:21.354621 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:21.355195 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:21.668295 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:21.668319 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:21.674594 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0521 19:43:21.674618 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0521 19:43:24.913937 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0521 19:43:30.908482 3643 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0521 19:43:44.882371 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0521 19:43:47.791423 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=69 I0521 19:44:16.891799 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0521 19:44:49.796202 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 I0521 19:44:53.876171 3643 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0521 19:45:42.789514 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=143 I0521 19:45:52.897777 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0521 19:46:08.563087 3643 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:46:19.620225 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0521 19:46:19.620330 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-gtkgg IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-gtkgg: no pod IPs found I0521 19:46:25.048164 3643 namespace.go:98] [tsf] adding namespace I0521 19:46:25.049045 3643 namespace.go:102] [tsf] adding namespace took 856.648µs I0521 19:46:25.063773 3643 namespace.go:142] [tsf] updating namespace I0521 19:46:25.073877 3643 namespace.go:142] [tsf] updating namespace I0521 19:46:27.913588 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0521 19:46:28.989949 3643 namespace.go:98] [tsf-keycloak] adding namespace I0521 19:46:28.990788 3643 namespace.go:102] [tsf-keycloak] adding namespace took 819.707µs I0521 19:46:28.995502 3643 namespace.go:98] [tsf-tas] adding namespace I0521 19:46:28.995554 3643 namespace.go:98] [rhtpa-operator] adding namespace I0521 19:46:28.995685 3643 namespace.go:98] [cert-manager-operator] adding namespace I0521 19:46:28.995733 3643 namespace.go:98] [konflux-operator] adding namespace I0521 19:46:28.996349 3643 namespace.go:102] [tsf-tas] adding namespace took 831.686µs I0521 19:46:28.996850 3643 namespace.go:102] [rhtpa-operator] adding namespace took 1.279585ms I0521 19:46:28.997495 3643 namespace.go:102] [cert-manager-operator] adding namespace took 1.791918ms I0521 19:46:28.997790 3643 namespace.go:98] [tsf-tpa] adding namespace I0521 19:46:28.998001 3643 namespace.go:102] [konflux-operator] adding namespace took 2.256278ms I0521 19:46:28.998540 3643 namespace.go:102] [tsf-tpa] adding namespace took 737.7µs I0521 19:46:29.000518 3643 namespace.go:98] [konflux-ui] adding namespace I0521 19:46:29.000614 3643 namespace.go:98] [rhbk-operator] adding namespace I0521 19:46:29.001133 3643 namespace.go:102] [konflux-ui] adding namespace took 601.78µs I0521 19:46:29.001607 3643 namespace.go:102] [rhbk-operator] adding namespace took 977.043µs I0521 19:46:29.014120 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:46:29.029120 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:46:29.049166 3643 namespace.go:142] [tsf-tas] updating namespace I0521 19:46:29.062066 3643 namespace.go:142] [tsf-tas] updating namespace I0521 19:46:29.064706 3643 namespace.go:142] [konflux-operator] updating namespace I0521 19:46:29.064775 3643 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:46:29.064888 3643 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:46:29.073831 3643 namespace.go:142] [rhbk-operator] updating namespace I0521 19:46:29.073932 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:46:29.074550 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:46:29.099651 3643 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:46:29.111337 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:46:29.111459 3643 namespace.go:142] [tsf-tas] updating namespace I0521 19:46:29.117974 3643 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:46:29.124400 3643 namespace.go:142] [konflux-operator] updating namespace I0521 19:46:29.140346 3643 namespace.go:142] [konflux-operator] updating namespace I0521 19:46:29.145426 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:46:29.157462 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:46:29.167339 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:46:29.178831 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:46:29.189458 3643 namespace.go:142] [rhbk-operator] updating namespace I0521 19:46:29.206428 3643 namespace.go:142] [rhbk-operator] updating namespace I0521 19:46:29.211217 3643 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:46:29.222106 3643 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:46:29.792035 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=71 I0521 19:46:30.267436 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:46:30.276081 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:46:37.340751 3643 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:37.341291 3643 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:37.341322 3643 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:37.341330 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:46:37.341340 3643 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:37.341344 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:46:38.370077 3643 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:38.370103 3643 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:38.370120 3643 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:38.370134 3643 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51wn2gp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:43.654554 3643 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:43.655117 3643 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:43.655144 3643 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:43.655153 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:46:43.655163 3643 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:43.655167 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:46:44.671115 3643 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.671661 3643 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.671693 3643 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.671702 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:46:44.671716 3643 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.671721 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:46:44.683387 3643 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:44.683411 3643 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:44.683426 3643 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:44.683436 3643 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dq6qg9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:44.700977 3643 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.701450 3643 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.701473 3643 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.701481 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:46:44.701491 3643 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.701500 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:46:44.729645 3643 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.730129 3643 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.730152 3643 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.730160 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:46:44.730168 3643 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:44.730173 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:46:45.681073 3643 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:45.681581 3643 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:45.681607 3643 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:45.681616 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:46:45.681627 3643 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:46:45.681631 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:46:45.699617 3643 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.699637 3643 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.699652 3643 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.699662 3643 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d74qm62 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.734362 3643 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.734384 3643 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.734395 3643 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.734405 3643 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213685rn2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.761306 3643 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.761330 3643 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.761340 3643 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:45.761356 3643 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ezfblv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:46.713512 3643 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:46.713538 3643 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:46.713554 3643 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:46.713567 3643 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dp76g4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:49.274207 3643 namespace.go:142] [openshift-operators] updating namespace I0521 19:46:49.287919 3643 namespace.go:142] [openshift-operators] updating namespace W0521 19:46:50.744728 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0521 19:46:52.046168 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0521 19:46:52.061208 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0521 19:46:52.061270 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0521 19:46:52.074579 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0521 19:46:58.087541 3643 namespace.go:142] [openshift-operators] updating namespace I0521 19:47:05.102474 3643 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-fqnpn for pod on switch ip-10-0-2-167.ec2.internal I0521 19:47:05.102609 3643 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn I0521 19:47:05.159392 3643 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn] pod update took 56.810328ms I0521 19:47:05.159422 3643 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn] addLogicalPort annotation time took 56.844272ms I0521 19:47:05.160645 3643 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn] addLogicalPort took 58.186849ms, libovsdb time 871.314µs I0521 19:47:07.215256 3643 namespace.go:98] [cert-manager] adding namespace I0521 19:47:07.216081 3643 namespace.go:102] [cert-manager] adding namespace took 800.61µs I0521 19:47:07.242298 3643 namespace.go:142] [cert-manager] updating namespace I0521 19:47:07.250016 3643 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn c31c6b7703090eede6312eb557021d2a7a50ede1daa4a18d7359519dd76a6fe1 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn c31c6b7703090eede6312eb557021d2a7a50ede1daa4a18d7359519dd76a6fe1 network default NAD default NAD key ] I0521 19:47:07.259178 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-fqnpn, hostIfaceName: c31c6b7703090ee, network: default, NAD default, SandboxID: "c31c6b7703090eede6312eb557021d2a7a50ede1daa4a18d7359519dd76a6fe1", PCI device ID: , UID: "987c27d3-43d6-4066-9bea-4516d0645edc", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0521 19:47:07.264950 3643 namespace.go:142] [cert-manager] updating namespace I0521 19:47:07.309184 3643 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn c31c6b7703090eede6312eb557021d2a7a50ede1daa4a18d7359519dd76a6fe1 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-fqnpn c31c6b7703090eede6312eb557021d2a7a50ede1daa4a18d7359519dd76a6fe1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:59:dc:16:8a:c6\",\"name\":\"c31c6b7703090ee\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8ebf86a-2c7b-43e0-ba78-09b48c9b6e64\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:47:08.845011 3643 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-rltfn] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-rltfn for pod on switch ip-10-0-2-167.ec2.internal I0521 19:47:08.845102 3643 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-rltfn I0521 19:47:08.863446 3643 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-rltfn] pod update took 18.36787ms I0521 19:47:08.863472 3643 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-rltfn] addLogicalPort annotation time took 18.395769ms I0521 19:47:08.864920 3643 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-rltfn] addLogicalPort took 19.907939ms, libovsdb time 1.079722ms I0521 19:47:09.741763 3643 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-rltfn f9e245aceb4cf11009cd5345d7b3e9df54ca5a768be7a02677c165c122b4bbd9 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-rltfn f9e245aceb4cf11009cd5345d7b3e9df54ca5a768be7a02677c165c122b4bbd9 network default NAD default NAD key ] I0521 19:47:09.752043 3643 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-rltfn, hostIfaceName: f9e245aceb4cf11, network: default, NAD default, SandboxID: "f9e245aceb4cf11009cd5345d7b3e9df54ca5a768be7a02677c165c122b4bbd9", PCI device ID: , UID: "7ec6ff2d-00c7-4788-b85e-ade2cd0867ff", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0521 19:47:09.803377 3643 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-rltfn f9e245aceb4cf11009cd5345d7b3e9df54ca5a768be7a02677c165c122b4bbd9 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-rltfn f9e245aceb4cf11009cd5345d7b3e9df54ca5a768be7a02677c165c122b4bbd9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:17:9c:37:80:66\",\"name\":\"f9e245aceb4cf11\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e9fd5ff-ccbb-42a8-afcb-673e6d795616\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:47:09.976061 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0521 19:47:10.276238 3643 namespace.go:98] [trusted-artifact-signer] adding namespace I0521 19:47:10.277244 3643 namespace.go:102] [trusted-artifact-signer] adding namespace took 982.863µs I0521 19:47:10.348277 3643 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:47:10.411226 3643 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:47:10.851355 3643 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-6w9t5 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:47:10.851468 3643 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5 I0521 19:47:10.868871 3643 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5] pod update took 17.431038ms I0521 19:47:10.868895 3643 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5] addLogicalPort annotation time took 17.460801ms I0521 19:47:10.870578 3643 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5] addLogicalPort took 19.240798ms, libovsdb time 1.13534ms I0521 19:47:11.231643 3643 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5 fbaf44ab79bed4cad70ffeb96a832bd1061d675197351c5c141f306a6c0cf594 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5 fbaf44ab79bed4cad70ffeb96a832bd1061d675197351c5c141f306a6c0cf594 network default NAD default NAD key ] I0521 19:47:11.242513 3643 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-6w9t5, hostIfaceName: fbaf44ab79bed4c, network: default, NAD default, SandboxID: "fbaf44ab79bed4cad70ffeb96a832bd1061d675197351c5c141f306a6c0cf594", PCI device ID: , UID: "2d224a01-4592-43c8-b968-e8678dae8b29", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0521 19:47:11.294811 3643 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5 fbaf44ab79bed4cad70ffeb96a832bd1061d675197351c5c141f306a6c0cf594 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-6w9t5 fbaf44ab79bed4cad70ffeb96a832bd1061d675197351c5c141f306a6c0cf594 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:e0:b3:2c:29:b2\",\"name\":\"fbaf44ab79bed4c\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dbd7faf2-212b-4eeb-aba4-b7aad581545c\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:47:11.506545 3643 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7] creating logical port trusted-artifact-signer_cli-server-7d56d9fd65-x2cd7 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:47:11.506650 3643 kube.go:256] Updating pod trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7 I0521 19:47:11.522143 3643 pod.go:62] [trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7] pod update took 15.516251ms I0521 19:47:11.522171 3643 base_network_controller_pods.go:951] [default/trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7] addLogicalPort annotation time took 15.549249ms I0521 19:47:11.523661 3643 pods.go:271] [trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7] addLogicalPort took 17.136062ms, libovsdb time 1.015576ms I0521 19:47:11.789989 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0521 19:47:12.253652 3643 cni.go:452] [trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7 465ef7c5e75a8e3dea84b809170d520a7a4ffe4dca9c6927b4584c23be6b244f network default NAD default NAD key ] ADD starting CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7 465ef7c5e75a8e3dea84b809170d520a7a4ffe4dca9c6927b4584c23be6b244f network default NAD default NAD key ] I0521 19:47:12.263106 3643 helper_linux.go:556] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-7d56d9fd65-x2cd7, hostIfaceName: 465ef7c5e75a8e3, network: default, NAD default, SandboxID: "465ef7c5e75a8e3dea84b809170d520a7a4ffe4dca9c6927b4584c23be6b244f", PCI device ID: , UID: "94040044-f696-4b77-b591-ab16dd18aebc", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0521 19:47:12.314723 3643 cni.go:473] [trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7 465ef7c5e75a8e3dea84b809170d520a7a4ffe4dca9c6927b4584c23be6b244f network default NAD default NAD key default] ADD finished CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-x2cd7 465ef7c5e75a8e3dea84b809170d520a7a4ffe4dca9c6927b4584c23be6b244f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:bd:db:95:62:b5\",\"name\":\"465ef7c5e75a8e3\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/43a88a60-7762-4b39-b4d8-d2b47163f00f\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:47:13.012033 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0521 19:47:13.012049 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0521 19:47:14.937138 3643 namespace.go:98] [openshift-pipelines] adding namespace I0521 19:47:14.938098 3643 namespace.go:102] [openshift-pipelines] adding namespace took 941.121µs I0521 19:47:14.956376 3643 namespace.go:142] [openshift-pipelines] updating namespace I0521 19:47:16.676952 3643 namespace.go:142] [cert-manager] updating namespace I0521 19:47:16.867611 3643 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:47:17.309156 3643 namespace.go:142] [konflux-operator] updating namespace I0521 19:47:17.492691 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:47:17.723286 3643 namespace.go:142] [open-cluster-management-2qegn84g18ids9l2o4klqb2qr2p2km0b] updating namespace W0521 19:47:17.897146 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0521 19:47:18.101123 3643 namespace.go:142] [rhbk-operator] updating namespace I0521 19:47:18.243991 3643 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5] creating logical port rhbk-operator_rhbk-operator-6fc7dbc666-zxgv5 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:47:18.244106 3643 kube.go:256] Updating pod rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5 I0521 19:47:18.262182 3643 pod.go:62] [rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5] pod update took 18.115658ms I0521 19:47:18.262206 3643 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5] addLogicalPort annotation time took 18.142789ms I0521 19:47:18.263560 3643 pods.go:271] [rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5] addLogicalPort took 19.584669ms, libovsdb time 974.526µs I0521 19:47:18.375167 3643 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:47:18.608765 3643 cni.go:452] [rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5 f20b9adb35de87cc4365baab3805eb7aa9c19d05b85a560f5c01d783516d3507 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5 f20b9adb35de87cc4365baab3805eb7aa9c19d05b85a560f5c01d783516d3507 network default NAD default NAD key ] I0521 19:47:18.619063 3643 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-6fc7dbc666-zxgv5, hostIfaceName: f20b9adb35de87c, network: default, NAD default, SandboxID: "f20b9adb35de87cc4365baab3805eb7aa9c19d05b85a560f5c01d783516d3507", PCI device ID: , UID: "19238409-6ac6-4e10-9c49-08092a5bddcf", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0521 19:47:18.679606 3643 cni.go:473] [rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5 f20b9adb35de87cc4365baab3805eb7aa9c19d05b85a560f5c01d783516d3507 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-6fc7dbc666-zxgv5 f20b9adb35de87cc4365baab3805eb7aa9c19d05b85a560f5c01d783516d3507 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:a6:7d:14:6f:1a\",\"name\":\"f20b9adb35de87c\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd6f00ff-568a-477c-9bec-22a1ab15e31d\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:47:18.702290 3643 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:47:18.881984 3643 namespace.go:142] [tsf] updating namespace W0521 19:47:18.981388 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0521 19:47:18.981407 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0521 19:47:19.166758 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:47:19.351101 3643 namespace.go:142] [tsf-tas] updating namespace I0521 19:47:19.573398 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:47:19.789168 3643 namespace.go:142] [cert-manager] updating namespace I0521 19:47:19.808984 3643 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:47:19.841426 3643 namespace.go:142] [default] updating namespace I0521 19:47:19.856023 3643 namespace.go:142] [konflux-operator] updating namespace I0521 19:47:19.878122 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:47:19.897825 3643 namespace.go:142] [open-cluster-management-2qegn84g18ids9l2o4klqb2qr2p2km0b] updating namespace I0521 19:47:19.934471 3643 namespace.go:142] [openshift] updating namespace I0521 19:47:19.987208 3643 namespace.go:142] [rhbk-operator] updating namespace I0521 19:47:19.987336 3643 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:47:20.010138 3643 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:47:20.026281 3643 namespace.go:142] [tsf] updating namespace I0521 19:47:20.041033 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:47:20.068592 3643 namespace.go:142] [tsf-tas] updating namespace I0521 19:47:20.113168 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:47:20.196229 3643 namespace.go:142] [cert-manager] updating namespace I0521 19:47:20.277795 3643 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:47:20.388655 3643 namespace.go:142] [default] updating namespace I0521 19:47:20.488346 3643 namespace.go:142] [konflux-operator] updating namespace I0521 19:47:20.567398 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:47:20.660837 3643 namespace.go:142] [open-cluster-management-2qegn84g18ids9l2o4klqb2qr2p2km0b] updating namespace I0521 19:47:20.733227 3643 namespace.go:142] [openshift] updating namespace I0521 19:47:20.790979 3643 namespace.go:142] [rhbk-operator] updating namespace I0521 19:47:20.918744 3643 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:47:21.004464 3643 namespace.go:142] [trusted-artifact-signer] updating namespace W0521 19:47:21.026296 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0521 19:47:21.192694 3643 namespace.go:142] [tsf] updating namespace I0521 19:47:21.367845 3643 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:47:21.591689 3643 namespace.go:142] [tsf-tas] updating namespace I0521 19:47:21.772250 3643 namespace.go:142] [tsf-tpa] updating namespace I0521 19:47:22.793648 3643 namespace.go:142] [openshift-pipelines] updating namespace I0521 19:47:33.529211 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-xbm94 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:47:33.529338 3643 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94 I0521 19:47:33.582339 3643 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94] pod update took 53.024968ms I0521 19:47:33.582362 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94] addLogicalPort annotation time took 53.053783ms I0521 19:47:33.583639 3643 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94] addLogicalPort took 54.440393ms, libovsdb time 914.005µs I0521 19:47:33.981395 3643 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:47:33.981952 3643 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:47:34.372025 3643 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:47:34.372048 3643 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:47:34.529479 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0521 19:47:34.529511 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0521 19:47:35.400885 3643 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94 8bcca6d32d319836b84763ee87d4d5c5d0ff570cd5d6249897f4550cbcae0097 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94 8bcca6d32d319836b84763ee87d4d5c5d0ff570cd5d6249897f4550cbcae0097 network default NAD default NAD key ] I0521 19:47:35.409990 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-xbm94, hostIfaceName: 8bcca6d32d31983, network: default, NAD default, SandboxID: "8bcca6d32d319836b84763ee87d4d5c5d0ff570cd5d6249897f4550cbcae0097", PCI device ID: , UID: "9f9222e6-099e-4d0a-b79b-747109376f93", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0521 19:47:35.460820 3643 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94 8bcca6d32d319836b84763ee87d4d5c5d0ff570cd5d6249897f4550cbcae0097 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-xbm94 8bcca6d32d319836b84763ee87d4d5c5d0ff570cd5d6249897f4550cbcae0097 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:70:c0:13:cf:d4\",\"name\":\"8bcca6d32d31983\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fda9ce3a-b797-43d1-9cbf-17b8bc5b174e\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:47:38.983139 3643 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0521 19:47:38.983161 3643 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0521 19:47:38.983167 3643 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0521 19:47:39.013327 3643 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:47:39.013343 3643 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:47:39.013347 3643 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:47:39.013351 3643 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0521 19:47:39.141081 3643 namespace.go:142] [openshift-pipelines] updating namespace I0521 19:47:40.877243 3643 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0521 19:47:45.902879 3643 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0521 19:47:47.894161 3643 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0521 19:47:48.110201 3643 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:47:48.110221 3643 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:47:48.110228 3643 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0521 19:47:50.902445 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 W0521 19:47:59.118729 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:47:59.118751 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:47:59.118758 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0521 19:48:06.015994 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-z9s8n for pod on switch ip-10-0-2-167.ec2.internal I0521 19:48:06.016110 3643 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n I0521 19:48:06.031241 3643 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n] pod update took 15.164538ms I0521 19:48:06.031264 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n] addLogicalPort annotation time took 15.19435ms I0521 19:48:06.032488 3643 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n] addLogicalPort took 16.516208ms, libovsdb time 844.443µs I0521 19:48:06.109614 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-hvwwh for pod on switch ip-10-0-2-167.ec2.internal I0521 19:48:06.109732 3643 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh I0521 19:48:06.127057 3643 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh] pod update took 17.361882ms I0521 19:48:06.127081 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh] addLogicalPort annotation time took 17.390878ms I0521 19:48:06.128286 3643 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh] addLogicalPort took 18.693089ms, libovsdb time 848.392µs I0521 19:48:06.371573 3643 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n 8e67d110f5a9d18c0e336d462b64603b525c23e9e2955998029c61de7cd0ed67 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n 8e67d110f5a9d18c0e336d462b64603b525c23e9e2955998029c61de7cd0ed67 network default NAD default NAD key ] I0521 19:48:06.380823 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-z9s8n, hostIfaceName: 8e67d110f5a9d18, network: default, NAD default, SandboxID: "8e67d110f5a9d18c0e336d462b64603b525c23e9e2955998029c61de7cd0ed67", PCI device ID: , UID: "0ef57673-4dd4-483c-a3dd-58e14d42e8b2", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0521 19:48:06.433837 3643 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n 8e67d110f5a9d18c0e336d462b64603b525c23e9e2955998029c61de7cd0ed67 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z9s8n 8e67d110f5a9d18c0e336d462b64603b525c23e9e2955998029c61de7cd0ed67 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:e5:16:fe:93:d4\",\"name\":\"8e67d110f5a9d18\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/434afc32-cc0d-4633-b01a-4e3703de1e5d\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:48:06.470884 3643 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh b39721ccb0198346de59a23d96f2f1775f421b8f27bf6a0c1487c95152882d69 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh b39721ccb0198346de59a23d96f2f1775f421b8f27bf6a0c1487c95152882d69 network default NAD default NAD key ] I0521 19:48:06.480426 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-hvwwh, hostIfaceName: b39721ccb019834, network: default, NAD default, SandboxID: "b39721ccb0198346de59a23d96f2f1775f421b8f27bf6a0c1487c95152882d69", PCI device ID: , UID: "45633a27-e89b-4e02-913d-cc7610d5971a", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0521 19:48:06.531024 3643 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh b39721ccb0198346de59a23d96f2f1775f421b8f27bf6a0c1487c95152882d69 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-hvwwh b39721ccb0198346de59a23d96f2f1775f421b8f27bf6a0c1487c95152882d69 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:b0:20:cd:07:f6\",\"name\":\"b39721ccb019834\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b015935a-20d0-4ec8-92bc-cfa19d3828e7\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:48:10.240221 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0521 19:48:11.170476 3643 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:48:11.171054 3643 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:48:11.191070 3643 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:48:11.191087 3643 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:48:11.194928 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0521 19:48:11.194956 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0521 19:48:12.894547 3643 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0521 19:48:12.895135 3643 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0521 19:48:12.895155 3643 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0521 19:48:12.897235 3643 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0521 19:48:20.195509 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0521 19:48:20.195528 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0521 19:48:20.200075 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0521 19:48:20.200092 3643 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0521 19:48:21.233592 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0521 19:48:28.041812 3643 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0521 19:48:42.827278 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-bw4rq for pod on switch ip-10-0-2-167.ec2.internal I0521 19:48:42.827368 3643 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq I0521 19:48:42.843978 3643 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq] pod update took 16.630698ms I0521 19:48:42.844001 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq] addLogicalPort annotation time took 16.658409ms I0521 19:48:42.845323 3643 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq] addLogicalPort took 18.060937ms, libovsdb time 942.021µs I0521 19:48:42.928808 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-r9d5x for pod on switch ip-10-0-2-167.ec2.internal I0521 19:48:42.928899 3643 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x I0521 19:48:42.945559 3643 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x] pod update took 16.687941ms I0521 19:48:42.945585 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x] addLogicalPort annotation time took 16.716747ms I0521 19:48:42.946919 3643 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x] addLogicalPort took 18.116996ms, libovsdb time 861.237µs I0521 19:48:43.787867 3643 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq deafbbbc4608fcd422657e48b6c9bb45a573386a3167be48e2468f642266d43f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq deafbbbc4608fcd422657e48b6c9bb45a573386a3167be48e2468f642266d43f network default NAD default NAD key ] I0521 19:48:43.797702 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-bw4rq, hostIfaceName: deafbbbc4608fcd, network: default, NAD default, SandboxID: "deafbbbc4608fcd422657e48b6c9bb45a573386a3167be48e2468f642266d43f", PCI device ID: , UID: "6d656a18-5faf-4541-bb1b-ec324f958189", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0521 19:48:43.848538 3643 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq deafbbbc4608fcd422657e48b6c9bb45a573386a3167be48e2468f642266d43f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-bw4rq deafbbbc4608fcd422657e48b6c9bb45a573386a3167be48e2468f642266d43f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:f7:b9:40:fb:69\",\"name\":\"deafbbbc4608fcd\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78cc57f9-4ddb-48d8-a818-50bb201f7559\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:48:43.883925 3643 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x ee102fb2de2209c5af493aa8d4a9a1d95bc3ed02daf4d4b7be05d49b543fc3be network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x ee102fb2de2209c5af493aa8d4a9a1d95bc3ed02daf4d4b7be05d49b543fc3be network default NAD default NAD key ] I0521 19:48:43.895086 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-r9d5x, hostIfaceName: ee102fb2de2209c, network: default, NAD default, SandboxID: "ee102fb2de2209c5af493aa8d4a9a1d95bc3ed02daf4d4b7be05d49b543fc3be", PCI device ID: , UID: "63916074-7c49-48b7-82bb-9cdea693065e", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0521 19:48:43.948044 3643 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x ee102fb2de2209c5af493aa8d4a9a1d95bc3ed02daf4d4b7be05d49b543fc3be network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-r9d5x ee102fb2de2209c5af493aa8d4a9a1d95bc3ed02daf4d4b7be05d49b543fc3be network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:f3:7a:9d:12:42\",\"name\":\"ee102fb2de2209c\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/982ff015-58e8-4dfb-b1d1-654b6eca128e\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:48:47.267252 3643 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0521 19:48:47.267271 3643 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0521 19:48:47.267276 3643 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0521 19:49:00.284438 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-5vlzz] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-5vlzz for pod on switch ip-10-0-2-167.ec2.internal I0521 19:49:00.284542 3643 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-5vlzz I0521 19:49:00.296726 3643 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-5vlzz] pod update took 12.219861ms I0521 19:49:00.296760 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-5vlzz] addLogicalPort annotation time took 12.252221ms I0521 19:49:00.298186 3643 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-5vlzz] addLogicalPort took 13.763162ms, libovsdb time 962.937µs I0521 19:49:00.645073 3643 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-5vlzz 47270002c8afedbb666fc39bf9ccbecef6766696264d0b3a6b90f9080546c31c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5vlzz 47270002c8afedbb666fc39bf9ccbecef6766696264d0b3a6b90f9080546c31c network default NAD default NAD key ] I0521 19:49:00.668392 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-5vlzz, hostIfaceName: 47270002c8afedb, network: default, NAD default, SandboxID: "47270002c8afedbb666fc39bf9ccbecef6766696264d0b3a6b90f9080546c31c", PCI device ID: , UID: "ba46f764-e3a4-4518-9b0e-13b481efa395", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0521 19:49:00.721133 3643 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-5vlzz 47270002c8afedbb666fc39bf9ccbecef6766696264d0b3a6b90f9080546c31c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5vlzz 47270002c8afedbb666fc39bf9ccbecef6766696264d0b3a6b90f9080546c31c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:ff:e4:18:62:ab\",\"name\":\"47270002c8afedb\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bce1c3b6-d4bf-4b2e-93ab-e67f47d41b74\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:49:03.434708 3643 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:03.435354 3643 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:03.473516 3643 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:03.473538 3643 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:03.479529 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0521 19:49:03.479575 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0521 19:49:05.966149 3643 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-9h8k2-726bk] creating logical port tsf-tas_ctlog-createtree-job-9h8k2-726bk for pod on switch ip-10-0-2-167.ec2.internal I0521 19:49:05.966282 3643 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-9h8k2-726bk I0521 19:49:05.983470 3643 pod.go:62] [tsf-tas/ctlog-createtree-job-9h8k2-726bk] pod update took 17.208577ms I0521 19:49:05.983492 3643 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-9h8k2-726bk] addLogicalPort annotation time took 17.235341ms I0521 19:49:05.984834 3643 pods.go:271] [tsf-tas/ctlog-createtree-job-9h8k2-726bk] addLogicalPort took 18.707353ms, libovsdb time 921.611µs I0521 19:49:06.212718 3643 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-4l8mc-lkr6m] creating logical port tsf-tas_rekor-createtree-job-4l8mc-lkr6m for pod on switch ip-10-0-2-167.ec2.internal I0521 19:49:06.212842 3643 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-4l8mc-lkr6m I0521 19:49:06.231100 3643 pod.go:62] [tsf-tas/rekor-createtree-job-4l8mc-lkr6m] pod update took 18.281782ms I0521 19:49:06.231122 3643 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-4l8mc-lkr6m] addLogicalPort annotation time took 18.311048ms I0521 19:49:06.232484 3643 pods.go:271] [tsf-tas/rekor-createtree-job-4l8mc-lkr6m] addLogicalPort took 19.786359ms, libovsdb time 954.387µs I0521 19:49:06.330372 3643 cni.go:452] [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key ] I0521 19:49:06.341366 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-9h8k2-726bk, hostIfaceName: 6784db2a0d89cfa, network: default, NAD default, SandboxID: "6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77", PCI device ID: , UID: "64a9c8d4-bc8f-466f-b0c4-cafd1086ca86", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0521 19:49:06.399575 3643 cni.go:473] [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:2c:9b:8f:59:64\",\"name\":\"6784db2a0d89cfa\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b9779c1-9abe-4a0c-a77d-585438c081e7\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:49:06.580136 3643 cni.go:452] [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key ] I0521 19:49:06.590352 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-4l8mc-lkr6m, hostIfaceName: e025711fc75fba5, network: default, NAD default, SandboxID: "e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da", PCI device ID: , UID: "e44e9a27-6bc4-48d7-be4a-be39a3a67790", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0521 19:49:06.644318 3643 cni.go:473] [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:0c:87:b9:ad:23\",\"name\":\"e025711fc75fba5\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2975c998-4492-4fc4-b77d-5ac259cb2bee\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:49:08.361241 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0521 19:49:08.361265 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0521 19:49:13.801662 3643 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-sg2ms of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:13.802257 3643 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-sg2ms of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:14.553807 3643 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-sg2ms of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:14.553829 3643 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-sg2ms of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:49:19.434332 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0521 19:49:19.484202 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0521 19:49:21.454407 3643 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0521 19:49:22.605313 3643 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-qts55-m8q2k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:22.605787 3643 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-qts55-m8q2k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0521 19:49:22.643997 3643 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0521 19:49:23.505287 3643 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-qts55-m8q2k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:23.505311 3643 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-qts55-m8q2k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:27.895459 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0521 19:49:34.774025 3643 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-k6ds2 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:49:34.774113 3643 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2 I0521 19:49:34.791983 3643 pod.go:62] [openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2] pod update took 17.892051ms I0521 19:49:34.792006 3643 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2] addLogicalPort annotation time took 17.918736ms I0521 19:49:34.793259 3643 pods.go:271] [openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2] addLogicalPort took 19.256265ms, libovsdb time 845.482µs I0521 19:49:36.632216 3643 cni.go:452] [openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2 4a45a6f708153b9de2bc04ce5b8653c6f83f5e27281ba8f0884d58fa7de08b73 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2 4a45a6f708153b9de2bc04ce5b8653c6f83f5e27281ba8f0884d58fa7de08b73 network default NAD default NAD key ] I0521 19:49:36.641375 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-k6ds2, hostIfaceName: 4a45a6f708153b9, network: default, NAD default, SandboxID: "4a45a6f708153b9de2bc04ce5b8653c6f83f5e27281ba8f0884d58fa7de08b73", PCI device ID: , UID: "2f43d76a-459d-4489-9863-bfa74b6fcb28", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0521 19:49:36.692099 3643 cni.go:473] [openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2 4a45a6f708153b9de2bc04ce5b8653c6f83f5e27281ba8f0884d58fa7de08b73 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-k6ds2 4a45a6f708153b9de2bc04ce5b8653c6f83f5e27281ba8f0884d58fa7de08b73 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:f4:95:e6:38:3a\",\"name\":\"4a45a6f708153b9\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b5a59302-f998-4ecd-b165-e2623ff82f96\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:49:53.781103 3643 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:53.781644 3643 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:53.801437 3643 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:53.801457 3643 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:53.806866 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0521 19:49:53.806890 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0521 19:50:02.830224 3643 obj_retry.go:498] Detected object tsf/patch-tekton-config-ncmr7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:02.830742 3643 obj_retry.go:498] Detected object tsf/patch-tekton-config-ncmr7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:03.853480 3643 obj_retry.go:492] Detected object tsf/patch-tekton-config-ncmr7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:03.853503 3643 obj_retry.go:492] Detected object tsf/patch-tekton-config-ncmr7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:03.961104 3643 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-sldg7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:03.961631 3643 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-sldg7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:04.831649 3643 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-sldg7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:04.831670 3643 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-sldg7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:12.849425 3643 cni.go:452] [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key ] I0521 19:50:12.928572 3643 cni.go:473] [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-4l8mc-lkr6m e025711fc75fba512021211483ccdd100e691864685e09f9052743a4382740da network default NAD default NAD key default], result "{}", err I0521 19:50:12.977317 3643 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-4l8mc-lkr6m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:12.977356 3643 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-4l8mc-lkr6m I0521 19:50:12.978605 3643 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-4l8mc-lkr6m, ips: 10.130.0.39 I0521 19:50:12.978631 3643 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-4l8mc-lkr6m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:13.870151 3643 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-4l8mc-lkr6m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:13.870173 3643 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-4l8mc-lkr6m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:13.888782 3643 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-clqft of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:13.889324 3643 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-clqft of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:14.915522 3643 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-clqft of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:14.915549 3643 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-clqft of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:14.954502 3643 obj_retry.go:492] Detected object tsf/patch-tekton-config-ncmr7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:14.954527 3643 obj_retry.go:492] Detected object tsf/patch-tekton-config-ncmr7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:14.963221 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-ncmr7 I0521 19:50:14.963244 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-ncmr7 I0521 19:50:17.511740 3643 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj] creating logical port tsf-tas_rekor-search-ui-77d75bc5d9-7nxvj for pod on switch ip-10-0-2-167.ec2.internal I0521 19:50:17.511827 3643 kube.go:256] Updating pod tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj I0521 19:50:17.526817 3643 pod.go:62] [tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj] pod update took 15.014851ms I0521 19:50:17.526840 3643 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj] addLogicalPort annotation time took 15.039585ms I0521 19:50:17.528175 3643 pods.go:271] [tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj] addLogicalPort took 16.45049ms, libovsdb time 887.545µs I0521 19:50:17.888131 3643 cni.go:452] [tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj ec2eb3e81f41c9f1423821da0d971493a08e315d4a2760fc9734c9313fe6cf18 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj ec2eb3e81f41c9f1423821da0d971493a08e315d4a2760fc9734c9313fe6cf18 network default NAD default NAD key ] I0521 19:50:17.897330 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-77d75bc5d9-7nxvj, hostIfaceName: ec2eb3e81f41c9f, network: default, NAD default, SandboxID: "ec2eb3e81f41c9f1423821da0d971493a08e315d4a2760fc9734c9313fe6cf18", PCI device ID: , UID: "408cf4fe-df32-4d2b-81e5-72c4eb643d2e", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0521 19:50:17.951553 3643 cni.go:473] [tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj ec2eb3e81f41c9f1423821da0d971493a08e315d4a2760fc9734c9313fe6cf18 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-77d75bc5d9-7nxvj ec2eb3e81f41c9f1423821da0d971493a08e315d4a2760fc9734c9313fe6cf18 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:3e:88:f6:46:15\",\"name\":\"ec2eb3e81f41c9f\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85beb0c0-c137-4756-bb6c-bc4161295a7b\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:50:19.889079 3643 cni.go:452] [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key ] I0521 19:50:19.917961 3643 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:19.918455 3643 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:19.939039 3643 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:19.939062 3643 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:19.944259 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0521 19:50:19.944282 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0521 19:50:19.972472 3643 cni.go:473] [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-9h8k2-726bk 6784db2a0d89cfa1588c4c8b998d6a44f16b8b000c7faeddc8c9935e9a7fce77 network default NAD default NAD key default], result "{}", err I0521 19:50:20.024856 3643 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9h8k2-726bk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:20.024891 3643 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-9h8k2-726bk I0521 19:50:20.026347 3643 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-9h8k2-726bk, ips: 10.130.0.38 I0521 19:50:20.026383 3643 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9h8k2-726bk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:20.911857 3643 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9h8k2-726bk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:20.911881 3643 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9h8k2-726bk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:21.461254 3643 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-v2zhr] creating logical port tsf-tpa_create-db-v2zhr for pod on switch ip-10-0-2-167.ec2.internal I0521 19:50:21.461351 3643 kube.go:256] Updating pod tsf-tpa/create-db-v2zhr I0521 19:50:21.497935 3643 kube.go:256] Updating pod tsf-tpa/create-db-v2zhr I0521 19:50:21.514130 3643 pod.go:62] [tsf-tpa/create-db-v2zhr] pod update took 52.801787ms I0521 19:50:21.514158 3643 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-v2zhr] addLogicalPort annotation time took 52.833533ms I0521 19:50:21.515684 3643 pods.go:271] [tsf-tpa/create-db-v2zhr] addLogicalPort took 54.44572ms, libovsdb time 999.584µs I0521 19:50:22.852086 3643 cni.go:452] [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key ] I0521 19:50:22.862764 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-v2zhr, hostIfaceName: cbe498136b23e7d, network: default, NAD default, SandboxID: "cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5", PCI device ID: , UID: "d111e560-46e1-4984-9d70-84e040019494", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0521 19:50:22.918098 3643 cni.go:473] [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:b8:b7:dc:44:06\",\"name\":\"cbe498136b23e7d\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/77d3ad89-64a6-4d31-8c84-07cac6174e1f\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:50:23.936757 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0521 19:50:23.958442 3643 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0521 19:50:24.832045 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0521 19:50:27.939380 3643 cni.go:452] [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key ] I0521 19:50:28.014921 3643 cni.go:473] [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-v2zhr cbe498136b23e7dc431a64cb56f38e7c183d5e83385af1d9a5412ae12ceaf3e5 network default NAD default NAD key default], result "{}", err I0521 19:50:28.064519 3643 obj_retry.go:498] Detected object tsf-tpa/create-db-v2zhr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:28.064560 3643 pods.go:176] Deleting pod: tsf-tpa/create-db-v2zhr I0521 19:50:28.066021 3643 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-v2zhr, ips: 10.130.0.42 I0521 19:50:28.066049 3643 obj_retry.go:498] Detected object tsf-tpa/create-db-v2zhr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0521 19:50:28.861039 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0521 19:50:28.959705 3643 obj_retry.go:492] Detected object tsf-tpa/create-db-v2zhr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:28.959730 3643 obj_retry.go:492] Detected object tsf-tpa/create-db-v2zhr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:29.071574 3643 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-dp9hf] creating logical port tsf-tpa_migrate-db-dp9hf for pod on switch ip-10-0-2-167.ec2.internal I0521 19:50:29.071667 3643 kube.go:256] Updating pod tsf-tpa/migrate-db-dp9hf I0521 19:50:29.087556 3643 pod.go:62] [tsf-tpa/migrate-db-dp9hf] pod update took 15.912183ms I0521 19:50:29.087580 3643 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-dp9hf] addLogicalPort annotation time took 15.941499ms I0521 19:50:29.088841 3643 pods.go:271] [tsf-tpa/migrate-db-dp9hf] addLogicalPort took 17.284975ms, libovsdb time 833.36µs I0521 19:50:29.428739 3643 cni.go:452] [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key ] I0521 19:50:29.439411 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-dp9hf, hostIfaceName: 287d50f7433fda7, network: default, NAD default, SandboxID: "287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0", PCI device ID: , UID: "01111e2e-a084-4f1e-be24-70e8777bd000", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0521 19:50:29.499143 3643 cni.go:473] [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:bf:58:78:30:9e\",\"name\":\"287d50f7433fda7\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38098d7b-5cbb-41ba-97d1-a9b323afd3ec\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:50:31.969724 3643 cni.go:452] [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key ] I0521 19:50:32.065206 3643 cni.go:473] [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-dp9hf 287d50f7433fda79933bce78b7d1cb2ce60e3b59aea60690b470832eb24e9ff0 network default NAD default NAD key default], result "{}", err I0521 19:50:32.117200 3643 obj_retry.go:498] Detected object tsf-tpa/migrate-db-dp9hf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:32.117238 3643 pods.go:176] Deleting pod: tsf-tpa/migrate-db-dp9hf I0521 19:50:32.118767 3643 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-dp9hf, ips: 10.130.0.43 I0521 19:50:32.118790 3643 obj_retry.go:498] Detected object tsf-tpa/migrate-db-dp9hf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:33.140030 3643 obj_retry.go:492] Detected object tsf-tpa/migrate-db-dp9hf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:33.140058 3643 obj_retry.go:492] Detected object tsf-tpa/migrate-db-dp9hf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:50:35.953844 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0521 19:50:36.299018 3643 base_network_controller_pods.go:487] [default/tsf-tpa/server-94577c4f9-56pr9] creating logical port tsf-tpa_server-94577c4f9-56pr9 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:50:36.299184 3643 kube.go:256] Updating pod tsf-tpa/server-94577c4f9-56pr9 I0521 19:50:36.317709 3643 pod.go:62] [tsf-tpa/server-94577c4f9-56pr9] pod update took 18.555708ms I0521 19:50:36.317736 3643 base_network_controller_pods.go:951] [default/tsf-tpa/server-94577c4f9-56pr9] addLogicalPort annotation time took 18.588505ms I0521 19:50:36.319135 3643 pods.go:271] [tsf-tpa/server-94577c4f9-56pr9] addLogicalPort took 20.123467ms, libovsdb time 973.218µs I0521 19:50:37.356649 3643 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-nscbc] creating logical port tsf-tpa_importer-86bcdf477b-nscbc for pod on switch ip-10-0-2-167.ec2.internal I0521 19:50:37.356746 3643 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-nscbc I0521 19:50:37.372974 3643 pod.go:62] [tsf-tpa/importer-86bcdf477b-nscbc] pod update took 16.253566ms I0521 19:50:37.372999 3643 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-nscbc] addLogicalPort annotation time took 16.283408ms I0521 19:50:37.374255 3643 pods.go:271] [tsf-tpa/importer-86bcdf477b-nscbc] addLogicalPort took 17.622371ms, libovsdb time 852.258µs I0521 19:50:40.077808 3643 obj_retry.go:498] Detected object tsf-tpa/create-importers-hn2vf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:40.078416 3643 obj_retry.go:498] Detected object tsf-tpa/create-importers-hn2vf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:41.103320 3643 obj_retry.go:492] Detected object tsf-tpa/create-importers-hn2vf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:41.103345 3643 obj_retry.go:492] Detected object tsf-tpa/create-importers-hn2vf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:47.762118 3643 cni.go:452] [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key ] I0521 19:50:47.771694 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-94577c4f9-56pr9, hostIfaceName: 984c1534dc84a27, network: default, NAD default, SandboxID: "984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489", PCI device ID: , UID: "f18472b1-56c1-49e1-979b-27bde2b36511", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0521 19:50:47.825850 3643 cni.go:473] [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:d9:b2:86:4c:c4\",\"name\":\"984c1534dc84a27\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f13d69d-ddb7-4f1b-b313-a366babefd5b\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:50:47.917746 3643 cni.go:452] [tsf-tpa/importer-86bcdf477b-nscbc 1b4e1369834552a57d052c9c04750941058dc4bba0a72ca757c212230dec216d network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-nscbc 1b4e1369834552a57d052c9c04750941058dc4bba0a72ca757c212230dec216d network default NAD default NAD key ] I0521 19:50:47.927660 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-nscbc, hostIfaceName: 1b4e1369834552a, network: default, NAD default, SandboxID: "1b4e1369834552a57d052c9c04750941058dc4bba0a72ca757c212230dec216d", PCI device ID: , UID: "5dbd4b7a-ca1d-4ce1-917a-ae6c94b458da", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0521 19:50:47.982448 3643 cni.go:473] [tsf-tpa/importer-86bcdf477b-nscbc 1b4e1369834552a57d052c9c04750941058dc4bba0a72ca757c212230dec216d network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-nscbc 1b4e1369834552a57d052c9c04750941058dc4bba0a72ca757c212230dec216d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:7b:d5:6e:52:76\",\"name\":\"1b4e1369834552a\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9cb03322-9883-41eb-93c8-a50b87afefe5\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0521 19:50:48.050057 3643 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0521 19:50:49.076706 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0521 19:50:51.803798 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=539 I0521 19:50:54.242582 3643 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-47sgc-bnrqq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:54.243199 3643 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-47sgc-bnrqq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:55.270697 3643 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-47sgc-bnrqq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:55.270726 3643 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-47sgc-bnrqq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:51:00.128885 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0521 19:51:06.224799 3643 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:06.225420 3643 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:06.267784 3643 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:06.267809 3643 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:06.272495 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0521 19:51:06.272518 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0521 19:51:07.039923 3643 namespace.go:98] [default-tenant] adding namespace I0521 19:51:07.040936 3643 namespace.go:102] [default-tenant] adding namespace took 991.738µs I0521 19:51:07.058173 3643 namespace.go:98] [integration-service] adding namespace I0521 19:51:07.059017 3643 namespace.go:102] [integration-service] adding namespace took 822.342µs I0521 19:51:07.062208 3643 namespace.go:98] [image-controller] adding namespace I0521 19:51:07.062844 3643 namespace.go:102] [image-controller] adding namespace took 619.359µs I0521 19:51:07.078603 3643 namespace.go:142] [default-tenant] updating namespace I0521 19:51:07.090041 3643 namespace.go:98] [build-service] adding namespace I0521 19:51:07.090851 3643 namespace.go:102] [build-service] adding namespace took 795.4µs I0521 19:51:07.131817 3643 namespace.go:142] [default-tenant] updating namespace I0521 19:51:07.153259 3643 namespace.go:142] [integration-service] updating namespace I0521 19:51:07.236816 3643 namespace.go:142] [image-controller] updating namespace I0521 19:51:07.306532 3643 namespace.go:142] [integration-service] updating namespace I0521 19:51:07.375618 3643 namespace.go:142] [build-service] updating namespace I0521 19:51:07.380389 3643 namespace.go:142] [image-controller] updating namespace I0521 19:51:07.440974 3643 namespace.go:142] [build-service] updating namespace I0521 19:51:07.690948 3643 namespace.go:142] [build-service] updating namespace I0521 19:51:07.716126 3643 namespace.go:142] [integration-service] updating namespace I0521 19:51:07.747109 3643 namespace.go:98] [release-service] adding namespace I0521 19:51:07.747983 3643 namespace.go:102] [release-service] adding namespace took 855.615µs I0521 19:51:07.797713 3643 namespace.go:142] [build-service] updating namespace I0521 19:51:07.814939 3643 namespace.go:142] [konflux-ui] updating namespace I0521 19:51:07.864257 3643 namespace.go:98] [konflux-info] adding namespace I0521 19:51:07.865101 3643 namespace.go:102] [konflux-info] adding namespace took 821.848µs I0521 19:51:07.897102 3643 namespace.go:98] [namespace-lister] adding namespace I0521 19:51:07.897822 3643 namespace.go:102] [namespace-lister] adding namespace took 704.675µs I0521 19:51:07.941247 3643 namespace.go:98] [enterprise-contract-service] adding namespace I0521 19:51:07.941975 3643 namespace.go:102] [enterprise-contract-service] adding namespace took 705.904µs I0521 19:51:07.965130 3643 namespace.go:142] [image-controller] updating namespace I0521 19:51:08.095014 3643 namespace.go:142] [release-service] updating namespace I0521 19:51:08.242240 3643 namespace.go:142] [default-tenant] updating namespace I0521 19:51:08.319219 3643 namespace.go:98] [segment-bridge] adding namespace I0521 19:51:08.319986 3643 namespace.go:102] [segment-bridge] adding namespace took 749.331µs I0521 19:51:08.397080 3643 namespace.go:142] [konflux-info] updating namespace I0521 19:51:08.458106 3643 namespace.go:98] [konflux-cli] adding namespace I0521 19:51:08.458265 3643 namespace.go:142] [default-tenant] updating namespace I0521 19:51:08.459005 3643 namespace.go:102] [konflux-cli] adding namespace took 879.904µs I0521 19:51:08.746582 3643 namespace.go:142] [konflux-info] updating namespace I0521 19:51:08.746695 3643 namespace.go:142] [namespace-lister] updating namespace I0521 19:51:08.769599 3643 namespace.go:142] [release-service] updating namespace I0521 19:51:08.884050 3643 namespace.go:142] [namespace-lister] updating namespace I0521 19:51:08.926048 3643 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:51:09.028053 3643 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:51:09.101365 3643 namespace.go:142] [segment-bridge] updating namespace I0521 19:51:09.181280 3643 namespace.go:142] [segment-bridge] updating namespace I0521 19:51:09.263094 3643 namespace.go:142] [konflux-cli] updating namespace I0521 19:51:09.306778 3643 namespace.go:142] [konflux-cli] updating namespace I0521 19:51:09.463397 3643 namespace.go:142] [image-controller] updating namespace I0521 19:51:09.683572 3643 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0521 19:51:09.684205 3643 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0521 19:51:09.684222 3643 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0521 19:51:09.686775 3643 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0521 19:51:09.711651 3643 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0521 19:51:09.711671 3643 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0521 19:51:09.712384 3643 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0521 19:51:09.958319 3643 namespace.go:142] [build-service] updating namespace I0521 19:51:09.986975 3643 namespace.go:142] [integration-service] updating namespace I0521 19:51:10.000233 3643 namespace.go:142] [default-tenant] updating namespace I0521 19:51:10.024339 3643 namespace.go:142] [image-controller] updating namespace I0521 19:51:10.051092 3643 namespace.go:142] [integration-service] updating namespace I0521 19:51:10.300121 3643 namespace.go:142] [build-service] updating namespace I0521 19:51:10.507080 3643 namespace.go:142] [default-tenant] updating namespace I0521 19:51:10.643652 3643 namespace.go:142] [image-controller] updating namespace I0521 19:51:10.735176 3643 namespace.go:142] [integration-service] updating namespace I0521 19:51:10.891852 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0521 19:51:12.184350 3643 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:51:12.437053 3643 namespace.go:142] [konflux-cli] updating namespace I0521 19:51:12.807170 3643 namespace.go:142] [konflux-info] updating namespace I0521 19:51:13.085033 3643 namespace.go:142] [namespace-lister] updating namespace I0521 19:51:13.276283 3643 base_network_controller_pods.go:487] [default/openshift-console/console-6667b8b6d-6zlrt] creating logical port openshift-console_console-6667b8b6d-6zlrt for pod on switch ip-10-0-2-167.ec2.internal I0521 19:51:13.276375 3643 kube.go:256] Updating pod openshift-console/console-6667b8b6d-6zlrt I0521 19:51:13.340267 3643 pod.go:62] [openshift-console/console-6667b8b6d-6zlrt] pod update took 63.915672ms I0521 19:51:13.340292 3643 base_network_controller_pods.go:951] [default/openshift-console/console-6667b8b6d-6zlrt] addLogicalPort annotation time took 63.947825ms I0521 19:51:13.341534 3643 pods.go:271] [openshift-console/console-6667b8b6d-6zlrt] addLogicalPort took 65.272898ms, libovsdb time 855.245µs I0521 19:51:13.447175 3643 namespace.go:142] [release-service] updating namespace I0521 19:51:13.643209 3643 cni.go:452] [openshift-console/console-6667b8b6d-6zlrt c90bb23162ea2aedc8f303ff7b16a5e46365a6fb63996d5f1f3325121d763226 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6667b8b6d-6zlrt c90bb23162ea2aedc8f303ff7b16a5e46365a6fb63996d5f1f3325121d763226 network default NAD default NAD key ] I0521 19:51:13.652611 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6667b8b6d-6zlrt, hostIfaceName: c90bb23162ea2ae, network: default, NAD default, SandboxID: "c90bb23162ea2aedc8f303ff7b16a5e46365a6fb63996d5f1f3325121d763226", PCI device ID: , UID: "3fcf77ff-c5f4-40e8-a462-a3358497c73c", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0521 19:51:13.661071 3643 namespace.go:142] [segment-bridge] updating namespace I0521 19:51:13.705789 3643 cni.go:473] [openshift-console/console-6667b8b6d-6zlrt c90bb23162ea2aedc8f303ff7b16a5e46365a6fb63996d5f1f3325121d763226 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6667b8b6d-6zlrt c90bb23162ea2aedc8f303ff7b16a5e46365a6fb63996d5f1f3325121d763226 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:7c:0d:3d:1f:53\",\"name\":\"c90bb23162ea2ae\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6ae83f57-4629-4069-b28d-8d199a55b1ec\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:51:13.824016 3643 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:51:13.853067 3643 namespace.go:142] [konflux-cli] updating namespace I0521 19:51:13.875388 3643 namespace.go:142] [konflux-info] updating namespace I0521 19:51:13.895073 3643 namespace.go:142] [namespace-lister] updating namespace I0521 19:51:13.906099 3643 namespace.go:142] [release-service] updating namespace I0521 19:51:13.942030 3643 namespace.go:142] [segment-bridge] updating namespace I0521 19:51:14.018106 3643 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:51:14.075517 3643 namespace.go:142] [konflux-cli] updating namespace I0521 19:51:14.135859 3643 namespace.go:142] [konflux-info] updating namespace W0521 19:51:14.152119 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0521 19:51:14.208338 3643 namespace.go:142] [namespace-lister] updating namespace I0521 19:51:14.373208 3643 namespace.go:142] [release-service] updating namespace I0521 19:51:14.560074 3643 namespace.go:142] [segment-bridge] updating namespace I0521 19:51:15.902452 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 W0521 19:51:18.169920 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0521 19:51:19.187293 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0521 19:51:20.201315 3643 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0521 19:51:21.263723 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0521 19:51:21.263739 3643 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0521 19:51:24.232320 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0521 19:51:24.234994 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0521 19:51:24.246982 3643 util.go:959] Failed to build node endpoints for node ip-10-0-2-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:51:24.255858 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0521 19:51:24.259826 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0521 19:51:24.295511 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0521 19:51:24.295533 3643 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0521 19:51:24.338319 3643 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0521 19:51:24.338403 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service I0521 19:51:35.635280 3643 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-dghrz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:35.635925 3643 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-dghrz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:36.324270 3643 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-dghrz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:36.324303 3643 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-dghrz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:36.328690 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-dghrz I0521 19:51:36.328714 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-dghrz I0521 19:51:36.886816 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0521 19:51:39.113739 3643 obj_retry.go:498] Detected object openshift-console/console-6b48447b9f-dgc2l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:39.114316 3643 obj_retry.go:498] Detected object openshift-console/console-6b48447b9f-dgc2l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:39.344783 3643 obj_retry.go:492] Detected object openshift-console/console-6b48447b9f-dgc2l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:39.344807 3643 obj_retry.go:492] Detected object openshift-console/console-6b48447b9f-dgc2l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:39.348409 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b48447b9f-dgc2l I0521 19:51:39.348436 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b48447b9f-dgc2l I0521 19:51:49.619391 3643 obj_retry.go:498] Detected object openshift-console/console-6b48447b9f-5jpgk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:49.619955 3643 obj_retry.go:498] Detected object openshift-console/console-6b48447b9f-5jpgk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:50.416700 3643 obj_retry.go:492] Detected object openshift-console/console-6b48447b9f-5jpgk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:50.416721 3643 obj_retry.go:492] Detected object openshift-console/console-6b48447b9f-5jpgk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:50.423419 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b48447b9f-5jpgk I0521 19:51:50.423447 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b48447b9f-5jpgk I0521 19:51:52.919309 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0521 19:52:15.917597 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0521 19:52:31.458393 3643 obj_retry.go:498] Detected object konflux-ui/proxy-74d45566-wlp26 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:52:31.459013 3643 obj_retry.go:498] Detected object konflux-ui/proxy-74d45566-wlp26 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:52:31.679701 3643 obj_retry.go:492] Detected object konflux-ui/proxy-74d45566-wlp26 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:52:31.679726 3643 obj_retry.go:492] Detected object konflux-ui/proxy-74d45566-wlp26 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:52:31.687464 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-74d45566-wlp26 I0521 19:52:31.687492 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-74d45566-wlp26 I0521 19:52:39.878381 3643 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:52:39.878974 3643 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:52:39.895508 3643 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:52:39.895526 3643 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:52:39.901953 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0521 19:52:39.901978 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0521 19:53:17.912297 3643 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0521 19:53:19.795455 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=92 I0521 19:53:32.880352 3643 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0521 19:53:42.224578 3643 namespace.go:98] [managed-tenant-ddruc] adding namespace I0521 19:53:42.225603 3643 namespace.go:102] [managed-tenant-ddruc] adding namespace took 1.000391ms I0521 19:53:42.254973 3643 namespace.go:142] [managed-tenant-ddruc] updating namespace I0521 19:53:42.287718 3643 namespace.go:142] [managed-tenant-ddruc] updating namespace I0521 19:53:43.277346 3643 namespace.go:142] [managed-tenant-ddruc] updating namespace I0521 19:53:43.406987 3643 namespace.go:142] [managed-tenant-ddruc] updating namespace I0521 19:53:43.508055 3643 namespace.go:142] [managed-tenant-ddruc] updating namespace I0521 19:53:46.289121 3643 obj_retry.go:492] Detected object tsf-tpa/create-db-v2zhr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:46.289147 3643 obj_retry.go:492] Detected object tsf-tpa/create-db-v2zhr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:46.296547 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-v2zhr I0521 19:53:46.296579 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-v2zhr I0521 19:53:47.808230 3643 obj_retry.go:334] Retry object setup: *v1.Pod tsf-tpa/create-db-nptbm I0521 19:53:47.808259 3643 obj_retry.go:398] Adding new object: *v1.Pod tsf-tpa/create-db-nptbm I0521 19:53:47.808299 3643 obj_retry.go:409] Retry add failed for *v1.Pod tsf-tpa/create-db-nptbm, will try again later: failed to obtain IPs to add remote pod tsf-tpa/create-db-nptbm: suppressed error logged: pod tsf-tpa/create-db-nptbm: no pod IPs found I0521 19:53:58.357390 3643 obj_retry.go:498] Detected object tsf-tpa/create-db-nptbm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:58.358035 3643 obj_retry.go:498] Detected object tsf-tpa/create-db-nptbm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:59.382871 3643 obj_retry.go:492] Detected object tsf-tpa/create-db-nptbm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:59.382900 3643 obj_retry.go:492] Detected object tsf-tpa/create-db-nptbm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:59.417707 3643 obj_retry.go:492] Detected object tsf-tpa/migrate-db-dp9hf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:59.417733 3643 obj_retry.go:492] Detected object tsf-tpa/migrate-db-dp9hf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:59.423382 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-dp9hf I0521 19:53:59.423409 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-dp9hf I0521 19:54:02.393124 3643 obj_retry.go:498] Detected object tsf-tpa/migrate-db-rcwwv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:02.393738 3643 obj_retry.go:498] Detected object tsf-tpa/migrate-db-rcwwv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:02.533752 3643 obj_retry.go:492] Detected object tsf-tpa/migrate-db-rcwwv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:02.533774 3643 obj_retry.go:492] Detected object tsf-tpa/migrate-db-rcwwv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:02.570674 3643 obj_retry.go:492] Detected object tsf-tpa/create-importers-hn2vf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:02.570691 3643 obj_retry.go:492] Detected object tsf-tpa/create-importers-hn2vf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:02.574788 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-hn2vf I0521 19:54:02.574811 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-hn2vf I0521 19:54:05.406716 3643 obj_retry.go:498] Detected object tsf-tpa/create-importers-7v6sn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:05.407352 3643 obj_retry.go:498] Detected object tsf-tpa/create-importers-7v6sn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:05.692216 3643 obj_retry.go:492] Detected object tsf-tpa/create-importers-7v6sn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:05.692237 3643 obj_retry.go:492] Detected object tsf-tpa/create-importers-7v6sn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:05.892656 3643 base_network_controller_pods.go:487] [default/tsf-tpa/server-598d99f679-q4vl6] creating logical port tsf-tpa_server-598d99f679-q4vl6 for pod on switch ip-10-0-2-167.ec2.internal I0521 19:54:05.892755 3643 kube.go:256] Updating pod tsf-tpa/server-598d99f679-q4vl6 I0521 19:54:05.914932 3643 pod.go:62] [tsf-tpa/server-598d99f679-q4vl6] pod update took 22.204582ms I0521 19:54:05.914952 3643 base_network_controller_pods.go:951] [default/tsf-tpa/server-598d99f679-q4vl6] addLogicalPort annotation time took 22.230047ms I0521 19:54:05.916261 3643 pods.go:271] [tsf-tpa/server-598d99f679-q4vl6] addLogicalPort took 23.621377ms, libovsdb time 861.191µs I0521 19:54:06.584181 3643 cni.go:452] [tsf-tpa/server-598d99f679-q4vl6 5adc4a5e880f12a80fdbbffe756e1f76c16304d74169ce694eb16943d7009ab9 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-598d99f679-q4vl6 5adc4a5e880f12a80fdbbffe756e1f76c16304d74169ce694eb16943d7009ab9 network default NAD default NAD key ] I0521 19:54:06.593648 3643 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-598d99f679-q4vl6, hostIfaceName: 5adc4a5e880f12a, network: default, NAD default, SandboxID: "5adc4a5e880f12a80fdbbffe756e1f76c16304d74169ce694eb16943d7009ab9", PCI device ID: , UID: "7de46337-e2ae-4532-ad94-fa252ecb8ea5", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0521 19:54:06.647675 3643 cni.go:473] [tsf-tpa/server-598d99f679-q4vl6 5adc4a5e880f12a80fdbbffe756e1f76c16304d74169ce694eb16943d7009ab9 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-598d99f679-q4vl6 5adc4a5e880f12a80fdbbffe756e1f76c16304d74169ce694eb16943d7009ab9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:50:57:0c:c7:12\",\"name\":\"5adc4a5e880f12a\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e0f0f5a-eb4a-4b6a-807f-16340a5e05fa\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0521 19:54:08.309478 3643 cni.go:452] [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key ] I0521 19:54:08.399846 3643 cni.go:473] [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/server-94577c4f9-56pr9 984c1534dc84a277647ef1e1e7d5b62d6ac67a23d3366b84c833bb08f7702489 network default NAD default NAD key default], result "{}", err I0521 19:54:08.460320 3643 obj_retry.go:498] Detected object tsf-tpa/server-94577c4f9-56pr9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:08.460354 3643 pods.go:176] Deleting pod: tsf-tpa/server-94577c4f9-56pr9 I0521 19:54:08.461629 3643 pods.go:236] Attempting to release IPs for pod: tsf-tpa/server-94577c4f9-56pr9, ips: 10.130.0.44 I0521 19:54:08.461652 3643 obj_retry.go:498] Detected object tsf-tpa/server-94577c4f9-56pr9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:09.272845 3643 obj_retry.go:492] Detected object tsf-tpa/server-94577c4f9-56pr9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:09.272867 3643 obj_retry.go:492] Detected object tsf-tpa/server-94577c4f9-56pr9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:09.277263 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/server-94577c4f9-56pr9 I0521 19:54:09.277289 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/server-94577c4f9-56pr9