+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-88.ec2.internal ++ [[ -n ip-10-0-2-88.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-88.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' I0421 03:01:18.254870480 - disable conntrack on geneve port + echo 'I0421 03:01:18.254870480 - disable conntrack on geneve port' + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0421 03:01:18.308120532 - starting ovnkube-node' I0421 03:01:18.308120532 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-88.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-88.ec2.internal --init-node ip-10-0-2-88.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-88.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0421 03:01:18.365773 3808 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0421 03:01:18.365830 3808 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-d3f818dbdd.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0421 03:01:18.367747 3808 kube.go:431] Waiting for certificate I0421 03:01:18.367778 3808 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0421 03:01:18.367818 3808 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0421 03:01:18.368565 3808 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0421 03:01:18.394951 3808 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0421 03:01:18.394966 3808 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0421 03:01:18.397371 3808 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0421 03:01:18.399767 3808 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-gspc7" I0421 03:01:18.410274 3808 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-gspc7" I0421 03:01:18.410361 3808 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0421 03:01:19.367860 3808 kube.go:438] Certificate found I0421 03:01:19.368619 3808 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0421 03:01:19.369877 3808 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0421 03:01:19.423906 3808 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-22 02:56:18 +0000 UTC" deadline="2026-04-21 21:45:26.409502401 +0000 UTC" I0421 03:01:19.423937 3808 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h44m6.985575203s" I0421 03:01:19.444919 3808 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0421 03:01:19.463665 3808 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0421 03:01:19.463728 3808 server.go:101] MetricServer registers OVS metrics I0421 03:01:19.464048 3808 server.go:105] MetricServer registers OVN DB metrics I0421 03:01:19.464094 3808 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0421 03:01:19.464191 3808 factory.go:561] Starting watch factory I0421 03:01:19.464294 3808 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464318 3808 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464318 3808 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464339 3808 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464320 3808 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464374 3808 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464388 3808 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464399 3808 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464656 3808 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464682 3808 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464951 3808 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.464977 3808 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.467776 3808 controller_manager.go:370] Starting the ovnkube controller I0421 03:01:19.467789 3808 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-88.ec2.internal I0421 03:01:19.467846 3808 controller_manager.go:395] NBDB zone sync took: 48.035µs I0421 03:01:19.467859 3808 factory.go:561] Starting watch factory I0421 03:01:19.469378 3808 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0421 03:01:19.469399 3808 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0421 03:01:19.482786 3808 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.483031 3808 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0421 03:01:19.483055 3808 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0421 03:01:19.483173 3808 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.483720 3808 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.484073 3808 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.486626 3808 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.488556 3808 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0421 03:01:19.488594 3808 server.go:109] MetricServer registers OVN Controller metrics I0421 03:01:19.492604 3808 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0421 03:01:19.494498 3808 server.go:113] MetricServer registers OVN Northd metrics I0421 03:01:19.499127 3808 metrics.go:519] OVN Metrics Server starts to run ... I0421 03:01:19.570022 3808 factory.go:1977] *v1.Namespace informer cache synced successfully I0421 03:01:19.570041 3808 factory.go:1977] *v1.Node informer cache synced successfully I0421 03:01:19.570045 3808 factory.go:1977] *v1.Service informer cache synced successfully I0421 03:01:19.570049 3808 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0421 03:01:19.570052 3808 factory.go:1977] *v1.Pod informer cache synced successfully I0421 03:01:19.570055 3808 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0421 03:01:19.570132 3808 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0421 03:01:19.570157 3808 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0421 03:01:19.570166 3808 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0421 03:01:19.570185 3808 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0421 03:01:19.573469 3808 factory.go:1977] *v1.Namespace informer cache synced successfully I0421 03:01:19.573481 3808 factory.go:1977] *v1.Node informer cache synced successfully I0421 03:01:19.573485 3808 factory.go:1977] *v1.Service informer cache synced successfully I0421 03:01:19.573488 3808 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0421 03:01:19.573492 3808 factory.go:1977] *v1.Pod informer cache synced successfully I0421 03:01:19.573495 3808 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0421 03:01:19.593818 3808 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0421 03:01:19.613090 3808 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0421 03:01:19.675875 3808 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0421 03:01:19.675890 3808 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0421 03:01:19.676026 3808 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.676071 3808 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.679516 3808 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0421 03:01:19.679529 3808 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0421 03:01:19.696295 3808 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.782009 3808 factory.go:1977] *v1.EgressIP informer cache synced successfully I0421 03:01:19.782091 3808 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.782106 3808 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.785562 3808 factory.go:1977] *v1.EgressIP informer cache synced successfully I0421 03:01:19.801896 3808 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.888806 3808 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0421 03:01:19.888913 3808 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.888933 3808 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.891349 3808 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0421 03:01:19.907748 3808 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.995412 3808 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0421 03:01:19.995500 3808 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.995519 3808 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:19.997853 3808 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0421 03:01:20.015741 3808 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.101936 3808 factory.go:1977] *v1.EgressService informer cache synced successfully I0421 03:01:20.102024 3808 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.102040 3808 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.104176 3808 factory.go:1977] *v1.EgressService informer cache synced successfully I0421 03:01:20.104867 3808 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.107352 3808 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0421 03:01:20.107505 3808 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0421 03:01:20.107529 3808 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0421 03:01:20.109751 3808 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0421 03:01:20.132293 3808 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0421 03:01:20.213291 3808 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0421 03:01:20.213397 3808 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.213417 3808 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.213433 3808 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.213445 3808 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.215905 3808 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0421 03:01:20.216531 3808 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.258352 3808 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0421 03:01:20.320242 3808 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0421 03:01:20.320256 3808 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0421 03:01:20.320265 3808 factory.go:680] Watch Factory start up complete, took: 856.078709ms I0421 03:01:20.320335 3808 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0421 03:01:20.321598 3808 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0421 03:01:20.321610 3808 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0421 03:01:20.321618 3808 factory.go:680] Watch Factory start up complete, took: 853.758085ms I0421 03:01:20.321625 3808 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-88.ec2.internal" zone I0421 03:01:20.362893 3808 default_node_network_controller.go:710] Initializing the default node network controller I0421 03:01:20.435986 3808 udn_isolation.go:101] Starting UDN host isolation manager I0421 03:01:20.442243 3808 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0421 03:01:20.518812 3808 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0421 03:01:20.518934 3808 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0421 03:01:20.518949 3808 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0421 03:01:20.551656 3808 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0421 03:01:20.551718 3808 default_node_network_controller.go:809] Node ip-10-0-2-88.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0421 03:01:20.557614 3808 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.88"] k8s.ovn.org/zone-name:ip-10-0-2-88.ec2.internal] on node ip-10-0-2-88.ec2.internal I0421 03:01:20.581130 3808 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0421 03:01:20.587046 3808 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0421 03:01:20.587201 3808 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0421 03:01:20.587366 3808 gateway_init.go:263] Preparing Gateway I0421 03:01:20.587379 3808 gateway_shared_intf.go:1687] Creating new gateway I0421 03:01:20.671737 3808 iptables.go:108] Creating table: filter chain: FORWARD I0421 03:01:20.678358 3808 iptables.go:108] Creating table: filter chain: OUTPUT I0421 03:01:20.685177 3808 gateway_shared_intf.go:1816] Gateway Creation Complete I0421 03:01:20.685196 3808 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-88.ec2.internal","mac-address":"0e:0b:2d:81:b5:77","ip-addresses":["10.0.2.88/24"],"ip-address":"10.0.2.88/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ad447213-f432-497e-b8c6-ca1cae6dea8b k8s.ovn.org/node-encap-ips:["10.0.2.88"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.88/24"} k8s.ovn.org/zone-name:ip-10-0-2-88.ec2.internal] on node ip-10-0-2-88.ec2.internal I0421 03:01:20.703720 3808 default_node_network_controller.go:519] Node connection status = connected I0421 03:01:20.822664 3808 controller_manager.go:429] Waiting for node in zone sync took: 501.02349ms I0421 03:01:20.828273 3808 controller_manager.go:611] Switching to transit router for layer2 networks I0421 03:01:20.828287 3808 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-88.ec2.internal I0421 03:01:20.843457 3808 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0421 03:01:20.850918 3808 controller_manager.go:290] SCTP support detected in OVN I0421 03:01:20.851528 3808 services_controller.go:69] Creating services controller for network=default I0421 03:01:20.851641 3808 udn_enabled_svc.go:80] Waiting for service informer to sync I0421 03:01:20.851740 3808 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0421 03:01:20.851779 3808 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0421 03:01:20.851792 3808 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0421 03:01:20.851801 3808 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0421 03:01:20.852241 3808 udn_enabled_svc.go:95] Performing full resync I0421 03:01:20.852380 3808 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0421 03:01:20.852430 3808 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0421 03:01:20.852482 3808 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 67.803µs I0421 03:01:20.852502 3808 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 63.363µs I0421 03:01:20.852510 3808 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 79.199µs I0421 03:01:20.852518 3808 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0421 03:01:20.852525 3808 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0421 03:01:20.852536 3808 default_network_controller.go:325] Starting the default network controller I0421 03:01:20.852554 3808 udn_enabled_svc.go:99] Waiting for handler to sync I0421 03:01:20.852563 3808 udn_enabled_svc.go:104] Starting worker I0421 03:01:20.852613 3808 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 13.108µs I0421 03:01:20.862694 3808 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0421 03:01:20.862751 3808 repair.go:33] Syncing exgw routes took 43.046µs I0421 03:01:20.862780 3808 default_network_controller.go:414] Starting all the Watchers... I0421 03:01:20.862935 3808 namespace.go:98] [openshift-cluster-version] adding namespace I0421 03:01:20.862937 3808 namespace.go:98] [openshift-kube-apiserver] adding namespace I0421 03:01:20.862952 3808 namespace.go:98] [openshift-package-operator] adding namespace I0421 03:01:20.862957 3808 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0421 03:01:20.862967 3808 namespace.go:98] [openshift-backplane-cse] adding namespace I0421 03:01:20.862970 3808 namespace.go:98] [openshift-service-ca-operator] adding namespace I0421 03:01:20.862966 3808 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0421 03:01:20.862972 3808 namespace.go:98] [dedicated-admin] adding namespace I0421 03:01:20.862979 3808 namespace.go:98] [openshift-dns-operator] adding namespace I0421 03:01:20.862982 3808 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0421 03:01:20.862986 3808 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0421 03:01:20.862996 3808 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0421 03:01:20.862982 3808 namespace.go:98] [openshift-operators-redhat] adding namespace I0421 03:01:20.863005 3808 namespace.go:98] [openshift-backplane-cee] adding namespace I0421 03:01:20.863009 3808 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0421 03:01:20.863719 3808 namespace.go:102] [openshift-cluster-version] adding namespace took 771.578µs I0421 03:01:20.863735 3808 namespace.go:98] [openshift-host-network] adding namespace I0421 03:01:20.864294 3808 namespace.go:102] [openshift-host-network] adding namespace took 551.576µs I0421 03:01:20.864309 3808 namespace.go:98] [openshift-network-node-identity] adding namespace I0421 03:01:20.864828 3808 namespace.go:102] [openshift-network-node-identity] adding namespace took 512.614µs I0421 03:01:20.864841 3808 namespace.go:98] [openshift-config] adding namespace I0421 03:01:20.865459 3808 namespace.go:102] [openshift-kube-apiserver] adding namespace took 2.509123ms I0421 03:01:20.865472 3808 namespace.go:98] [openshift-network-operator] adding namespace I0421 03:01:20.866000 3808 namespace.go:102] [openshift-package-operator] adding namespace took 3.040484ms I0421 03:01:20.866010 3808 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0421 03:01:20.866551 3808 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 3.584098ms I0421 03:01:20.866567 3808 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0421 03:01:20.867132 3808 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.158952ms I0421 03:01:20.867170 3808 namespace.go:98] [openshift-etcd] adding namespace I0421 03:01:20.867640 3808 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 4.663388ms I0421 03:01:20.867650 3808 namespace.go:98] [openshift-ingress-operator] adding namespace I0421 03:01:20.868091 3808 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.116184ms I0421 03:01:20.868106 3808 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0421 03:01:20.868525 3808 namespace.go:102] [dedicated-admin] adding namespace took 5.545811ms I0421 03:01:20.868538 3808 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0421 03:01:20.869040 3808 namespace.go:102] [openshift-dns-operator] adding namespace took 6.054663ms I0421 03:01:20.869054 3808 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0421 03:01:20.869902 3808 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.912268ms I0421 03:01:20.869916 3808 namespace.go:98] [kube-system] adding namespace I0421 03:01:20.870457 3808 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.464522ms I0421 03:01:20.870471 3808 namespace.go:98] [openshift-dns] adding namespace I0421 03:01:20.870892 3808 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.891771ms I0421 03:01:20.870902 3808 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0421 03:01:20.871372 3808 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.365166ms I0421 03:01:20.871383 3808 namespace.go:98] [open-cluster-management-2pq9drt9viog8225llnhb4eduucokg20] adding namespace I0421 03:01:20.871830 3808 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.814382ms I0421 03:01:20.871845 3808 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0421 03:01:20.872281 3808 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 9.26294ms I0421 03:01:20.872294 3808 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0421 03:01:20.872705 3808 namespace.go:102] [openshift-config] adding namespace took 7.859932ms I0421 03:01:20.872715 3808 namespace.go:98] [openshift-network-console] adding namespace I0421 03:01:20.873100 3808 namespace.go:102] [openshift-network-operator] adding namespace took 7.622151ms I0421 03:01:20.873113 3808 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0421 03:01:20.873559 3808 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.543154ms I0421 03:01:20.873572 3808 namespace.go:98] [openshift-machine-api] adding namespace I0421 03:01:20.873955 3808 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.383689ms I0421 03:01:20.873966 3808 namespace.go:98] [openshift-machine-config-operator] adding namespace I0421 03:01:20.874353 3808 namespace.go:102] [openshift-etcd] adding namespace took 7.177227ms I0421 03:01:20.874364 3808 namespace.go:98] [openshift-ingress] adding namespace I0421 03:01:20.874765 3808 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.10925ms I0421 03:01:20.874780 3808 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0421 03:01:20.875180 3808 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.068812ms I0421 03:01:20.875191 3808 namespace.go:98] [openshift-backplane-tam] adding namespace I0421 03:01:20.875584 3808 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.039732ms I0421 03:01:20.875596 3808 namespace.go:98] [openshift-console] adding namespace I0421 03:01:20.875973 3808 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.914175ms I0421 03:01:20.875985 3808 namespace.go:98] [default] adding namespace I0421 03:01:20.876361 3808 namespace.go:102] [kube-system] adding namespace took 6.44018ms I0421 03:01:20.876372 3808 namespace.go:98] [openshift-infra] adding namespace I0421 03:01:20.876788 3808 namespace.go:102] [openshift-dns] adding namespace took 6.310556ms I0421 03:01:20.876801 3808 namespace.go:98] [openshift-image-registry] adding namespace I0421 03:01:20.877213 3808 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.305614ms I0421 03:01:20.877224 3808 namespace.go:98] [kube-public] adding namespace I0421 03:01:20.877621 3808 namespace.go:102] [open-cluster-management-2pq9drt9viog8225llnhb4eduucokg20] adding namespace took 6.233773ms I0421 03:01:20.877633 3808 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0421 03:01:20.878017 3808 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.168234ms I0421 03:01:20.878028 3808 namespace.go:98] [openshift-backplane-csm] adding namespace I0421 03:01:20.878464 3808 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.165943ms I0421 03:01:20.878475 3808 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0421 03:01:20.878894 3808 namespace.go:102] [openshift-network-console] adding namespace took 6.174648ms I0421 03:01:20.879314 3808 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.19601ms I0421 03:01:20.879326 3808 namespace.go:98] [openshift-kube-scheduler] adding namespace I0421 03:01:20.879716 3808 namespace.go:102] [openshift-machine-api] adding namespace took 6.139336ms I0421 03:01:20.879727 3808 namespace.go:98] [openshift-console-user-settings] adding namespace I0421 03:01:20.880103 3808 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.132116ms I0421 03:01:20.880113 3808 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0421 03:01:20.880532 3808 namespace.go:102] [openshift-ingress] adding namespace took 6.162486ms I0421 03:01:20.880544 3808 namespace.go:98] [openshift-backplane-srep] adding namespace I0421 03:01:20.880931 3808 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.146497ms I0421 03:01:20.880942 3808 namespace.go:98] [openshift-node] adding namespace I0421 03:01:20.881304 3808 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.107376ms I0421 03:01:20.881314 3808 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0421 03:01:20.881725 3808 namespace.go:102] [openshift-console] adding namespace took 6.124507ms I0421 03:01:20.881736 3808 namespace.go:98] [openshift-apiserver-operator] adding namespace I0421 03:01:20.882099 3808 namespace.go:102] [default] adding namespace took 6.109789ms I0421 03:01:20.882111 3808 namespace.go:98] [openshift-network-diagnostics] adding namespace I0421 03:01:20.882523 3808 namespace.go:102] [openshift-infra] adding namespace took 6.146616ms I0421 03:01:20.882534 3808 namespace.go:98] [openshift-apiserver] adding namespace I0421 03:01:20.882895 3808 namespace.go:102] [openshift-image-registry] adding namespace took 6.088951ms I0421 03:01:20.882908 3808 namespace.go:98] [openshift-monitoring] adding namespace I0421 03:01:20.883366 3808 namespace.go:102] [kube-public] adding namespace took 6.137109ms I0421 03:01:20.883378 3808 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0421 03:01:20.883744 3808 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.105145ms I0421 03:01:20.883756 3808 namespace.go:98] [openshift-authentication-operator] adding namespace I0421 03:01:20.884135 3808 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.102073ms I0421 03:01:20.884158 3808 namespace.go:98] [openshift-backplane] adding namespace I0421 03:01:20.884572 3808 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.092318ms I0421 03:01:20.884583 3808 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0421 03:01:20.884901 3808 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.571506ms I0421 03:01:20.884909 3808 namespace.go:98] [kube-node-lease] adding namespace I0421 03:01:20.885388 3808 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.656608ms I0421 03:01:20.885401 3808 namespace.go:98] [openshift-multus] adding namespace I0421 03:01:20.885782 3808 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 5.663723ms I0421 03:01:20.885793 3808 namespace.go:98] [openshift-marketplace] adding namespace I0421 03:01:20.886163 3808 namespace.go:102] [openshift-backplane-srep] adding namespace took 5.615325ms I0421 03:01:20.886171 3808 namespace.go:98] [openshift-authentication] adding namespace I0421 03:01:20.886578 3808 namespace.go:102] [openshift-node] adding namespace took 5.629943ms I0421 03:01:20.886591 3808 namespace.go:98] [openshift-must-gather-operator] adding namespace I0421 03:01:20.886909 3808 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 5.589094ms I0421 03:01:20.886921 3808 namespace.go:98] [openshift-route-controller-manager] adding namespace W0421 03:01:20.887316 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-48xmk: pod openshift-network-diagnostics/network-check-target-48xmk: no pod IPs found W0421 03:01:20.887336 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-nw4z5: pod openshift-network-diagnostics/network-check-target-nw4z5: no pod IPs found W0421 03:01:20.887342 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-xl42b: pod openshift-network-diagnostics/network-check-target-xl42b: no pod IPs found I0421 03:01:20.887366 3808 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.624877ms I0421 03:01:20.887375 3808 namespace.go:98] [openshift-console-operator] adding namespace I0421 03:01:20.887790 3808 namespace.go:102] [openshift-network-diagnostics] adding namespace took 5.673952ms I0421 03:01:20.887802 3808 namespace.go:98] [openshift-insights] adding namespace I0421 03:01:20.888199 3808 namespace.go:102] [openshift-apiserver] adding namespace took 5.660694ms I0421 03:01:20.888208 3808 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0421 03:01:20.888616 3808 namespace.go:102] [openshift-monitoring] adding namespace took 5.704461ms I0421 03:01:20.888627 3808 namespace.go:98] [openshift-config-managed] adding namespace I0421 03:01:20.889007 3808 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.623463ms I0421 03:01:20.889019 3808 namespace.go:98] [openshift-customer-monitoring] adding namespace I0421 03:01:20.889400 3808 namespace.go:102] [openshift-authentication-operator] adding namespace took 5.639585ms I0421 03:01:20.889410 3808 namespace.go:98] [openshift-backplane-mobb] adding namespace I0421 03:01:20.889780 3808 namespace.go:102] [openshift-backplane] adding namespace took 5.617025ms I0421 03:01:20.889790 3808 namespace.go:98] [openshift-operators] adding namespace I0421 03:01:20.890586 3808 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.997832ms I0421 03:01:20.890596 3808 namespace.go:98] [openshift-config-operator] adding namespace W0421 03:01:20.891045 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-mcvv9: pod openshift-multus/network-metrics-daemon-mcvv9: no pod IPs found W0421 03:01:20.891056 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-v5wg8: pod openshift-multus/network-metrics-daemon-v5wg8: no pod IPs found W0421 03:01:20.891062 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-vspcz: pod openshift-multus/network-metrics-daemon-vspcz: no pod IPs found I0421 03:01:20.891097 3808 namespace.go:102] [kube-node-lease] adding namespace took 6.182484ms I0421 03:01:20.891583 3808 namespace.go:102] [openshift-multus] adding namespace took 6.176802ms I0421 03:01:20.891978 3808 namespace.go:102] [openshift-marketplace] adding namespace took 6.180047ms I0421 03:01:20.891990 3808 namespace.go:98] [openshift-logging] adding namespace I0421 03:01:20.892358 3808 namespace.go:102] [openshift-authentication] adding namespace took 6.182995ms I0421 03:01:20.892782 3808 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.18709ms I0421 03:01:20.893162 3808 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.236437ms I0421 03:01:20.893547 3808 namespace.go:102] [openshift-console-operator] adding namespace took 6.16689ms I0421 03:01:20.893925 3808 namespace.go:102] [openshift-insights] adding namespace took 6.118337ms I0421 03:01:20.893935 3808 namespace.go:98] [openshift-ingress-canary] adding namespace I0421 03:01:20.894336 3808 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.123141ms I0421 03:01:20.894719 3808 namespace.go:102] [openshift-config-managed] adding namespace took 6.086476ms I0421 03:01:20.895084 3808 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.060292ms I0421 03:01:20.895096 3808 namespace.go:98] [openshift] adding namespace I0421 03:01:20.895485 3808 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.069716ms I0421 03:01:20.895863 3808 namespace.go:102] [openshift-operators] adding namespace took 6.069052ms I0421 03:01:20.896238 3808 namespace.go:102] [openshift-config-operator] adding namespace took 5.637743ms I0421 03:01:20.896250 3808 namespace.go:98] [openshift-controller-manager] adding namespace I0421 03:01:20.896670 3808 namespace.go:102] [openshift-logging] adding namespace took 4.675165ms I0421 03:01:20.897016 3808 namespace.go:102] [openshift-ingress-canary] adding namespace took 3.075337ms I0421 03:01:20.897463 3808 namespace.go:102] [openshift] adding namespace took 2.362787ms I0421 03:01:20.897801 3808 namespace.go:102] [openshift-controller-manager] adding namespace took 1.546465ms W0421 03:01:20.897871 3808 master.go:471] Unable to parse remote node ip-10-0-0-186.ec2.internal chassis-id annotation. Chassis may be removed during sync W0421 03:01:20.897878 3808 master.go:471] Unable to parse remote node ip-10-0-1-71.ec2.internal chassis-id annotation. Chassis may be removed during sync I0421 03:01:20.897956 3808 master.go:548] Adding or Updating local node "ip-10-0-2-88.ec2.internal" for network "default" I0421 03:01:20.898002 3808 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-186.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-186.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-186.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-186.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-186.ec2.internal I0421 03:01:20.898004 3808 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-71.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-71.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-71.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-71.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal W0421 03:01:20.904586 3808 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-xl42b: pod openshift-network-diagnostics/network-check-target-xl42b: no pod IPs found W0421 03:01:20.904602 3808 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-vspcz: pod openshift-multus/network-metrics-daemon-vspcz: no pod IPs found I0421 03:01:20.905445 3808 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-88.ec2.internal for the network default I0421 03:01:20.905717 3808 zone_ic_handler.go:195] Time taken to create transit switch: 261.991µs I0421 03:01:20.906452 3808 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0421 03:01:20.906488 3808 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0421 03:01:20.906501 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0421 03:01:20.906508 3808 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0421 03:01:20.906527 3808 node_tracker.go:235] Node ip-10-0-1-71.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-71.ec2.internal" I0421 03:01:20.906540 3808 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:20.906553 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:20.906560 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-88.ec2.internal in network "default" I0421 03:01:20.906948 3808 node_tracker.go:169] Node ip-10-0-2-88.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:20.906956 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:20.906965 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-186.ec2.internal in network "default" I0421 03:01:20.906973 3808 node_tracker.go:235] Node ip-10-0-0-186.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-186.ec2.internal" I0421 03:01:20.906978 3808 node_tracker.go:169] Node ip-10-0-0-186.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:20.906982 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:20.923928 3808 default_network_controller.go:1025] Node "ip-10-0-0-186.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:21.006603 3808 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0421 03:01:21.006622 3808 services_controller.go:208] Setting up event handlers for services for network=default I0421 03:01:21.006754 3808 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0421 03:01:21.006818 3808 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0421 03:01:21.006836 3808 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0421 03:01:21.107291 3808 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0421 03:01:21.107318 3808 repair.go:60] Starting repairing loop for services I0421 03:01:21.107447 3808 repair.go:131] Deleted 0 stale service LBs I0421 03:01:21.107474 3808 repair.go:137] Deleted 0 stale Chassis Template Vars I0421 03:01:21.107498 3808 repair.go:62] Finished repairing loop for services: 183.844µs I0421 03:01:21.107536 3808 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0421 03:01:21.107551 3808 services_controller.go:254] Starting workers for network=default I0421 03:01:21.107954 3808 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-v5wg8, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-v5wg8: suppressed error logged: pod openshift-multus/network-metrics-daemon-v5wg8: no pod IPs found I0421 03:01:21.107972 3808 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-48xmk, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-48xmk: suppressed error logged: pod openshift-network-diagnostics/network-check-target-48xmk: no pod IPs found I0421 03:01:21.107995 3808 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xl42b] creating logical port openshift-network-diagnostics_network-check-target-xl42b for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:21.108009 3808 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-nw4z5, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-nw4z5: suppressed error logged: pod openshift-network-diagnostics/network-check-target-nw4z5: no pod IPs found I0421 03:01:21.108173 3808 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-mcvv9, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-mcvv9: suppressed error logged: pod openshift-multus/network-metrics-daemon-mcvv9: no pod IPs found I0421 03:01:21.108195 3808 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-xl42b I0421 03:01:21.108239 3808 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-vspcz] creating logical port openshift-multus_network-metrics-daemon-vspcz for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:21.108326 3808 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-vspcz I0421 03:01:21.126908 3808 pod.go:62] [openshift-network-diagnostics/network-check-target-xl42b] pod update took 18.822658ms I0421 03:01:21.126911 3808 pod.go:62] [openshift-multus/network-metrics-daemon-vspcz] pod update took 18.609034ms I0421 03:01:21.126929 3808 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-xl42b] addLogicalPort annotation time took 18.84769ms I0421 03:01:21.126936 3808 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-vspcz] addLogicalPort annotation time took 18.634186ms I0421 03:01:21.127838 3808 pods.go:271] [openshift-network-diagnostics/network-check-target-xl42b] addLogicalPort took 19.852776ms, libovsdb time 647.501µs I0421 03:01:21.128091 3808 pods.go:271] [openshift-multus/network-metrics-daemon-vspcz] addLogicalPort took 19.857884ms, libovsdb time 884.512µs I0421 03:01:21.128670 3808 admin_network_policy_controller.go:227] Starting controller default-network-controller I0421 03:01:21.128697 3808 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0421 03:01:21.128714 3808 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0421 03:01:21.128721 3808 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0421 03:01:21.128915 3808 repair.go:29] Repairing admin network policies took 188.693µs I0421 03:01:21.129028 3808 repair.go:92] Repairing baseline admin network policies took 103.445µs I0421 03:01:21.129608 3808 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0421 03:01:21.129627 3808 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0421 03:01:21.131648 3808 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0421 03:01:21.131673 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0421 03:01:21.131684 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0421 03:01:21.133817 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0421 03:01:21.133841 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0421 03:01:21.133853 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0421 03:01:21.135133 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0421 03:01:21.135173 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0421 03:01:21.135612 3808 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0421 03:01:21.135631 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0421 03:01:21.137699 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0421 03:01:21.137722 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0421 03:01:21.137732 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0421 03:01:21.139403 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0421 03:01:21.139421 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0421 03:01:21.139430 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0421 03:01:21.139992 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0421 03:01:21.140009 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0421 03:01:21.140016 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0421 03:01:21.141548 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0421 03:01:21.141577 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0421 03:01:21.141610 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0421 03:01:21.143981 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0421 03:01:21.144005 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0421 03:01:21.144015 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0421 03:01:21.144952 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0421 03:01:21.144977 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0421 03:01:21.144986 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0421 03:01:21.145360 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0421 03:01:21.145380 3808 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0421 03:01:21.145387 3808 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0421 03:01:21.145713 3808 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0421 03:01:21.145734 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0421 03:01:21.145743 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0421 03:01:21.146076 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0421 03:01:21.146095 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0421 03:01:21.146104 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0421 03:01:21.146745 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0421 03:01:21.146767 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0421 03:01:21.146779 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0421 03:01:21.147415 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0421 03:01:21.147433 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0421 03:01:21.147445 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0421 03:01:21.148853 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0421 03:01:21.148873 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0421 03:01:21.148884 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0421 03:01:21.150464 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0421 03:01:21.150487 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0421 03:01:21.150500 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0421 03:01:21.151650 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0421 03:01:21.151669 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0421 03:01:21.151677 3808 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0421 03:01:21.153534 3808 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0421 03:01:21.153552 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0421 03:01:21.153562 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0421 03:01:21.154130 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0421 03:01:21.154170 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0421 03:01:21.154182 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0421 03:01:21.155486 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0421 03:01:21.155508 3808 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0421 03:01:21.155523 3808 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0421 03:01:21.156667 3808 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0421 03:01:21.156685 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0421 03:01:21.156694 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0421 03:01:21.157224 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0421 03:01:21.157249 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0421 03:01:21.157260 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0421 03:01:21.157777 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0421 03:01:21.157794 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0421 03:01:21.157802 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0421 03:01:21.158383 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0421 03:01:21.158399 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0421 03:01:21.158406 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0421 03:01:21.158930 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0421 03:01:21.158952 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0421 03:01:21.158963 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0421 03:01:21.159504 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0421 03:01:21.159524 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0421 03:01:21.159532 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0421 03:01:21.160111 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0421 03:01:21.160133 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0421 03:01:21.160166 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0421 03:01:21.160861 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0421 03:01:21.160884 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0421 03:01:21.160900 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0421 03:01:21.162063 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0421 03:01:21.167316 3808 egressfirewall.go:374] Starting EgressFirewall controller I0421 03:01:21.167351 3808 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0421 03:01:21.167358 3808 controller.go:133] Adding controller egress-firewall-controller event handlers I0421 03:01:21.167394 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0421 03:01:21.167398 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0421 03:01:21.167412 3808 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0421 03:01:21.167416 3808 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0421 03:01:21.167730 3808 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0421 03:01:21.167748 3808 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0421 03:01:21.167754 3808 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0421 03:01:21.167763 3808 egressqos.go:193] Setting up event handlers for EgressQoS I0421 03:01:21.167786 3808 egressfirewall.go:723] Syncing node "ip-10-0-0-186.ec2.internal" for egress firewall I0421 03:01:21.167805 3808 egressfirewall.go:723] Syncing node "ip-10-0-1-71.ec2.internal" for egress firewall I0421 03:01:21.167811 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-88.ec2.internal" for egress firewall I0421 03:01:21.167920 3808 egressqos.go:245] Starting EgressQoS Controller I0421 03:01:21.167935 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0421 03:01:21.167946 3808 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0421 03:01:21.167953 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0421 03:01:21.167957 3808 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0421 03:01:21.167966 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0421 03:01:21.167972 3808 shared_informer.go:356] "Caches are synced" controller="egressqos" I0421 03:01:21.167976 3808 egressqos.go:259] Repairing EgressQoSes I0421 03:01:21.167981 3808 egressqos.go:399] Starting repairing loop for egressqos I0421 03:01:21.168075 3808 egressqos.go:401] Finished repairing loop for egressqos: 93.898µs I0421 03:01:21.168101 3808 egressservice_zone.go:132] Setting up event handlers for Egress Services I0421 03:01:21.168168 3808 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-88.ec2.internal I0421 03:01:21.168217 3808 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-88.ec2.internal : 51.399µs I0421 03:01:21.168270 3808 egressservice_zone.go:210] Starting Egress Services Controller I0421 03:01:21.168281 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0421 03:01:21.168288 3808 shared_informer.go:356] "Caches are synced" controller="egressservices" I0421 03:01:21.168297 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0421 03:01:21.168304 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0421 03:01:21.168313 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0421 03:01:21.168322 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0421 03:01:21.168329 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0421 03:01:21.168334 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0421 03:01:21.168337 3808 egressservice_zone.go:228] Repairing Egress Services I0421 03:01:21.168664 3808 master_controller.go:88] Starting Admin Policy Based Route Controller I0421 03:01:21.168676 3808 external_controller.go:278] Starting Admin Policy Based Route Controller I0421 03:01:21.168718 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-88.ec2.internal I0421 03:01:21.168734 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-88.ec2.internal: 18.251µs I0421 03:01:21.168743 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-186.ec2.internal I0421 03:01:21.168748 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-186.ec2.internal: 5.207µs I0421 03:01:21.168755 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0421 03:01:21.168762 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 5.957µs I0421 03:01:21.168957 3808 default_network_controller.go:585] Completing all the Watchers took 306.167534ms I0421 03:01:21.168974 3808 default_network_controller.go:589] Starting unidling controllers I0421 03:01:21.168990 3808 unidle.go:46] Registering OVN SB ControllerEvent handler I0421 03:01:21.168998 3808 unidle.go:63] Populating Initial ContollerEvent events I0421 03:01:21.169016 3808 unidle.go:79] Setting up event handlers for services I0421 03:01:21.169105 3808 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0421 03:01:21.204787 3808 default_node_network_controller.go:519] Node connection status = connected I0421 03:01:21.704443 3808 default_node_network_controller.go:519] Node connection status = connected I0421 03:01:22.204591 3808 default_node_network_controller.go:519] Node connection status = connected I0421 03:01:22.224277 3808 bridgeconfig.go:537] Gateway is ready I0421 03:01:22.224294 3808 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0421 03:01:22.235487 3808 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0421 03:01:22.236615 3808 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.88. Updating OVN encap IP. I0421 03:01:22.242004 3808 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.88 - it is already configured I0421 03:01:22.242018 3808 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.88/24:{} 172.20.0.1/32:{}]. Updating annotations. I0421 03:01:22.242268 3808 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.88/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-88.ec2.internal","mac-address":"0e:0b:2d:81:b5:77","ip-addresses":["10.0.2.88/24"],"ip-address":"10.0.2.88/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ad447213-f432-497e-b8c6-ca1cae6dea8b k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.88/24"}] on node ip-10-0-2-88.ec2.internal I0421 03:01:22.257208 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-88.ec2.internal I0421 03:01:22.257231 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-88.ec2.internal: 30.22µs I0421 03:01:22.257310 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-88.ec2.internal in network "default" I0421 03:01:22.257370 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-88.ec2.internal" for egress firewall I0421 03:01:22.257380 3808 node_tracker.go:169] Node ip-10-0-2-88.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:22.257392 3808 master.go:548] Adding or Updating local node "ip-10-0-2-88.ec2.internal" for network "default" I0421 03:01:22.257395 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:22.258079 3808 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.88 I0421 03:01:22.258407 3808 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-88.ec2.internal I0421 03:01:22.262920 3808 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-88.ec2.internal. Updating no re-route policies I0421 03:01:22.263005 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-hp2sc I0421 03:01:22.263015 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-48xmk I0421 03:01:22.263020 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-hp2sc I0421 03:01:22.263027 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-48xmk I0421 03:01:22.263046 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-hp2sc after 0 failed attempt(s) I0421 03:01:22.263005 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-bd5g2 I0421 03:01:22.263061 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-bd5g2 I0421 03:01:22.263063 3808 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-48xmk, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-48xmk: suppressed error logged: pod openshift-network-diagnostics/network-check-target-48xmk: no pod IPs found I0421 03:01:22.263072 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-bd5g2 after 0 failed attempt(s) I0421 03:01:22.263074 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-v5wg8 I0421 03:01:22.263073 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-7hhlm I0421 03:01:22.263084 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-lkkkk I0421 03:01:22.263086 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-88.ec2.internal I0421 03:01:22.263090 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-lkkkk I0421 03:01:22.263091 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-7hhlm I0421 03:01:22.263087 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-k845h I0421 03:01:22.263100 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-lkkkk after 0 failed attempt(s) I0421 03:01:22.263100 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-c9h6q I0421 03:01:22.263108 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-k845h I0421 03:01:22.263113 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-sv6sw I0421 03:01:22.263116 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-c9h6q I0421 03:01:22.263121 3808 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-c8pkv I0421 03:01:22.263129 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-sv6sw I0421 03:01:22.263155 3808 obj_retry.go:398] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-c8pkv I0421 03:01:22.263157 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-7hhlm after 0 failed attempt(s) I0421 03:01:22.263165 3808 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-88.ec2.internal I0421 03:01:22.263166 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-c9h6q after 0 failed attempt(s) I0421 03:01:22.263171 3808 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-lt6hk I0421 03:01:22.263177 3808 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-88.ec2.internal I0421 03:01:22.263177 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-bfrzr I0421 03:01:22.263178 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-sv6sw after 0 failed attempt(s) I0421 03:01:22.263184 3808 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-lt6hk I0421 03:01:22.263188 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-bfrzr I0421 03:01:22.263198 3808 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-88.ec2.internal after 0 failed attempt(s) I0421 03:01:22.263207 3808 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-lt6hk after 0 failed attempt(s) I0421 03:01:22.263210 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-bfrzr after 0 failed attempt(s) I0421 03:01:22.263080 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-v5wg8 I0421 03:01:22.263092 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-88.ec2.internal I0421 03:01:22.263249 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-k845h after 0 failed attempt(s) I0421 03:01:22.263178 3808 obj_retry.go:418] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-c8pkv after 0 failed attempt(s) I0421 03:01:22.263264 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xl42b I0421 03:01:22.263274 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-88.ec2.internal after 0 failed attempt(s) I0421 03:01:22.263314 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xl42b I0421 03:01:22.263256 3808 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-v5wg8, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-v5wg8: suppressed error logged: pod openshift-multus/network-metrics-daemon-v5wg8: no pod IPs found I0421 03:01:22.263350 3808 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xl42b] creating logical port openshift-network-diagnostics_network-check-target-xl42b for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:22.263171 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-vspcz I0421 03:01:22.263372 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-vspcz I0421 03:01:22.263406 3808 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-vspcz] creating logical port openshift-multus_network-metrics-daemon-vspcz for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:22.264195 3808 pods.go:271] [openshift-network-diagnostics/network-check-target-xl42b] addLogicalPort took 852.783µs, libovsdb time 437.72µs I0421 03:01:22.264207 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-xl42b after 0 failed attempt(s) I0421 03:01:22.264341 3808 pods.go:271] [openshift-multus/network-metrics-daemon-vspcz] addLogicalPort took 933.667µs, libovsdb time 496.474µs I0421 03:01:22.264357 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-vspcz after 0 failed attempt(s) I0421 03:01:22.272603 3808 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0421 03:01:22.285504 3808 iptables.go:108] Creating table: mangle chain: OUTPUT I0421 03:01:22.289470 3808 iptables.go:108] Creating table: nat chain: OUTPUT I0421 03:01:22.311447 3808 iptables.go:108] Creating table: nat chain: PREROUTING I0421 03:01:22.379537 3808 iptables.go:108] Creating table: filter chain: FORWARD I0421 03:01:22.398182 3808 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0421 03:01:22.398294 3808 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0421 03:01:22.398374 3808 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0421 03:01:22.398469 3808 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0421 03:01:22.398526 3808 gateway_init.go:325] Gateway and management port readiness took 1.69895561s I0421 03:01:22.398555 3808 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0421 03:01:22.398592 3808 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0421 03:01:22.398606 3808 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0421 03:01:22.398616 3808 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0421 03:01:22.398676 3808 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0421 03:01:22.398760 3808 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 52.589µs I0421 03:01:22.399380 3808 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0421 03:01:22.399406 3808 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.201µs I0421 03:01:22.399413 3808 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.641µs I0421 03:01:22.399418 3808 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0421 03:01:22.399429 3808 nad_controller.go:379] [node-nad-controller NAD controller]: started I0421 03:01:22.399435 3808 default_node_network_controller.go:899] Starting the default node network controller I0421 03:01:22.399442 3808 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0421 03:01:22.399530 3808 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.75µs I0421 03:01:22.399600 3808 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0421 03:01:22.399609 3808 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0421 03:01:22.399636 3808 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a03a20 0xc0002ef490 0xc0002ef500 0xc0002ef570 0xc0002ef5e0 0xc0002ef650 0xc0002ef6c0 0xc000b9c6e0 0xc0002ef7a0 0xc0002ef810 0xc000b242d0 0xc0064ff500 0xc0002ef880 map[0x2ff9080:0xc00250a480 0x2ff95c0:0xc00250a600 0x2ffa2e0:0xc001b939e0 0x2ffa580:0xc00250a420 0x2ffad60:0xc00250a4e0 0x2ffb000:0xc00250a540 0x2ffb2a0:0xc00250a5a0 0x2ffba80:0xc00250a660 0x2ffbd20:0xc00250a6c0 0x3026980:0xc000b99500 0x3026d20:0xc000b99560 0x3027800:0xc000a7aa80 0x3029160:0xc001cc4000 0x30298a0:0xc001b929c0 0x302a720:0xc00074ed80] 0xc000168b60 0} I0421 03:01:22.399694 3808 gateway.go:151] Starting gateway service sync I0421 03:01:22.400169 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0421 03:01:22.402569 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0421 03:01:22.441690 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0421 03:01:22.443072 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0421 03:01:22.444398 3808 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0421 03:01:22.478568 3808 gateway.go:168] Gateway service sync done. Time taken: 78.864125ms I0421 03:01:22.479226 3808 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32706 I0421 03:01:22.479428 3808 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a03a20 0xc0002ef490 0xc0002ef500 0xc0002ef570 0xc0002ef5e0 0xc0002ef650 0xc0002ef6c0 0xc000b9c6e0 0xc0002ef7a0 0xc0002ef810 0xc000b242d0 0xc0064ff500 0xc0002ef880 map[0x2ff9080:0xc00250a480 0x2ff95c0:0xc00250a600 0x2ffa2e0:0xc001b939e0 0x2ffa580:0xc00250a420 0x2ffad60:0xc00250a4e0 0x2ffb000:0xc00250a540 0x2ffb2a0:0xc00250a5a0 0x2ffba80:0xc00250a660 0x2ffbd20:0xc00250a6c0 0x3026980:0xc000b99500 0x3026d20:0xc000b99560 0x3027800:0xc000a7aa80 0x3029160:0xc001cc4000 0x30298a0:0xc001b929c0 0x302a720:0xc00074ed80] 0xc000168b60 0} I0421 03:01:22.479932 3808 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a03a20 0xc0002ef490 0xc0002ef500 0xc0002ef570 0xc0002ef5e0 0xc0002ef650 0xc0002ef6c0 0xc000b9c6e0 0xc0002ef7a0 0xc0002ef810 0xc000b242d0 0xc0064ff500 0xc0002ef880 map[0x2ff9080:0xc00250a480 0x2ff95c0:0xc00250a600 0x2ffa2e0:0xc001b939e0 0x2ffa580:0xc00250a420 0x2ffad60:0xc00250a4e0 0x2ffb000:0xc00250a540 0x2ffb2a0:0xc00250a5a0 0x2ffba80:0xc00250a660 0x2ffbd20:0xc00250a6c0 0x3026980:0xc000b99500 0x3026d20:0xc000b99560 0x3027800:0xc000a7aa80 0x3029160:0xc001cc4000 0x30298a0:0xc001b929c0 0x302a720:0xc00074ed80] 0xc000168b60 0} I0421 03:01:22.479971 3808 gateway.go:532] Reconciling gateway with updates I0421 03:01:22.485363 3808 gateway.go:339] Spawning Conntrack Rule Check Thread I0421 03:01:22.485442 3808 default_node_network_controller.go:944] Gateway and management port readiness took 85.826711ms I0421 03:01:22.485448 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0421 03:01:22.485453 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0421 03:01:22.485464 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0421 03:01:22.485463 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0421 03:01:22.485467 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0421 03:01:22.485480 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0421 03:01:22.485481 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0421 03:01:22.485492 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0421 03:01:22.485498 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0421 03:01:22.485498 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0421 03:01:22.485507 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0421 03:01:22.485512 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0421 03:01:22.485513 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0421 03:01:22.485519 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0421 03:01:22.485524 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0421 03:01:22.485527 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0421 03:01:22.485522 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0421 03:01:22.485538 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0421 03:01:22.485541 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0421 03:01:22.485544 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0421 03:01:22.485549 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0421 03:01:22.485550 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0421 03:01:22.485554 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0421 03:01:22.485562 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0421 03:01:22.485514 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0421 03:01:22.485563 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0421 03:01:22.485570 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0421 03:01:22.485571 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0421 03:01:22.485561 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0421 03:01:22.485579 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0421 03:01:22.485568 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0421 03:01:22.485578 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0421 03:01:22.485585 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0421 03:01:22.485587 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0421 03:01:22.485529 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0421 03:01:22.485596 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0421 03:01:22.485597 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0421 03:01:22.485601 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0421 03:01:22.485563 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0421 03:01:22.485607 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0421 03:01:22.485605 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0421 03:01:22.485606 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0421 03:01:22.485619 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0421 03:01:22.485621 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0421 03:01:22.485573 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0421 03:01:22.485625 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0421 03:01:22.485629 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0421 03:01:22.485575 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0421 03:01:22.485632 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0421 03:01:22.485640 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0421 03:01:22.485646 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0421 03:01:22.485539 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0421 03:01:22.485652 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0421 03:01:22.485651 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0421 03:01:22.485586 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0421 03:01:22.485648 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0421 03:01:22.485663 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0421 03:01:22.485658 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0421 03:01:22.485669 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0421 03:01:22.485674 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0421 03:01:22.485675 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0421 03:01:22.485680 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0421 03:01:22.485677 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0421 03:01:22.485686 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0421 03:01:22.485687 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0421 03:01:22.485581 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0421 03:01:22.485697 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0421 03:01:22.485441 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0421 03:01:22.485707 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0421 03:01:22.485656 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0421 03:01:22.485715 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0421 03:01:22.485589 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0421 03:01:22.485560 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0421 03:01:22.485763 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 32635 is already open, no action needed I0421 03:01:22.485612 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0421 03:01:22.485776 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 31771 is already open, no action needed I0421 03:01:22.485781 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0421 03:01:22.485595 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0421 03:01:22.485590 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0421 03:01:22.485615 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0421 03:01:22.485821 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0421 03:01:22.485609 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0421 03:01:22.485842 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0421 03:01:22.485859 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0421 03:01:22.485629 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0421 03:01:22.485630 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0421 03:01:22.485877 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0421 03:01:22.485881 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0421 03:01:22.485883 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0421 03:01:22.485893 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0421 03:01:22.485892 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0421 03:01:22.485640 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0421 03:01:22.485907 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0421 03:01:22.485632 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0421 03:01:22.485922 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0421 03:01:22.485927 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0421 03:01:22.485660 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0421 03:01:22.485898 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0421 03:01:22.485813 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0421 03:01:22.485624 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0421 03:01:22.485876 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0421 03:01:22.485631 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0421 03:01:22.485658 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0421 03:01:22.486067 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0421 03:01:22.485932 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0421 03:01:22.486599 3808 node_ip_handler_linux.go:160] Node IP manager is running W0421 03:01:22.523384 3808 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0421 03:01:22.523579 3808 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0421 03:01:22.523748 3808 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0421 03:01:22.531877 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0421 03:01:22.531905 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0421 03:01:22.531912 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0421 03:01:22.531936 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0421 03:01:22.531945 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0421 03:01:22.531951 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0421 03:01:22.576925 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0421 03:01:22.576952 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0421 03:01:22.576959 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0421 03:01:22.576965 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0421 03:01:22.576973 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0421 03:01:22.576991 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0421 03:01:22.577002 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0421 03:01:22.577010 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0421 03:01:22.577016 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0421 03:01:22.577021 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0421 03:01:22.577026 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0421 03:01:22.621256 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0421 03:01:22.621277 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0421 03:01:22.621285 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0421 03:01:22.621293 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0421 03:01:22.621301 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0421 03:01:22.621309 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0421 03:01:22.621314 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0421 03:01:22.621321 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0421 03:01:22.621325 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0421 03:01:22.621333 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0421 03:01:22.621338 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0421 03:01:22.621343 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0421 03:01:23.689981 3808 default_node_network_controller.go:1380] Starting node controller node sync W0421 03:01:23.690027 3808 egressip_healthcheck.go:76] Health checking using insecure connection I0421 03:01:23.690093 3808 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0421 03:01:23.701686 3808 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.681308ms I0421 03:01:23.701852 3808 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0421 03:01:23.702111 3808 egressservice_node.go:93] Setting up event handlers for Egress Services I0421 03:01:23.702282 3808 egressservice_node.go:183] Starting Egress Services Controller I0421 03:01:23.702298 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0421 03:01:23.702306 3808 shared_informer.go:356] "Caches are synced" controller="egressservices" I0421 03:01:23.702317 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0421 03:01:23.702323 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0421 03:01:23.702328 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0421 03:01:23.702333 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0421 03:01:23.702337 3808 egressservice_node.go:197] Repairing Egress Services I0421 03:01:23.712685 3808 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0421 03:01:23.712699 3808 external_controller.go:278] Starting Admin Policy Based Route Controller I0421 03:01:23.712928 3808 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0421 03:01:23.712942 3808 link_network_manager.go:119] Link manager is running I0421 03:01:23.712952 3808 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0421 03:01:23.713136 3808 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0421 03:01:23.713554 3808 vrf_manager.go:128] VRF manager is running I0421 03:01:23.713728 3808 node_controller_manager.go:437] Removing flows to drop GARP I0421 03:01:23.713737 3808 gateway.go:532] Reconciling gateway with updates I0421 03:01:23.739995 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0421 03:01:23.740010 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0421 03:01:23.740017 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0421 03:01:23.740027 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0421 03:01:23.740045 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 32635 is already open, no action needed I0421 03:01:23.740049 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0421 03:01:23.740022 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0421 03:01:23.740066 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0421 03:01:23.740069 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0421 03:01:23.740069 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0421 03:01:23.740083 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0421 03:01:23.740086 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0421 03:01:23.740092 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0421 03:01:23.740098 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0421 03:01:23.740110 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0421 03:01:23.740115 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0421 03:01:23.740116 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0421 03:01:23.740120 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0421 03:01:23.740112 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0421 03:01:23.740121 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0421 03:01:23.740127 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0421 03:01:23.740132 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0421 03:01:23.740132 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0421 03:01:23.740127 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0421 03:01:23.740162 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0421 03:01:23.740168 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0421 03:01:23.740171 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0421 03:01:23.740172 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0421 03:01:23.740173 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0421 03:01:23.740179 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0421 03:01:23.740015 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0421 03:01:23.740188 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0421 03:01:23.740191 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0421 03:01:23.740195 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0421 03:01:23.740194 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0421 03:01:23.740196 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0421 03:01:23.740201 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0421 03:01:23.740202 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0421 03:01:23.740118 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0421 03:01:23.740210 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0421 03:01:23.740210 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0421 03:01:23.740217 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0421 03:01:23.740218 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0421 03:01:23.740224 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0421 03:01:23.740213 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0421 03:01:23.740238 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0421 03:01:23.740241 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0421 03:01:23.740245 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0421 03:01:23.740246 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0421 03:01:23.740249 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0421 03:01:23.740253 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0421 03:01:23.740257 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0421 03:01:23.740259 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0421 03:01:23.740265 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0421 03:01:23.740266 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0421 03:01:23.740055 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 31771 is already open, no action needed I0421 03:01:23.740191 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0421 03:01:23.740101 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0421 03:01:23.740303 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0421 03:01:23.740308 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0421 03:01:23.740318 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0421 03:01:23.740088 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0421 03:01:23.740317 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0421 03:01:23.740334 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0421 03:01:23.740103 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0421 03:01:23.740339 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0421 03:01:23.740100 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0421 03:01:23.740344 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0421 03:01:23.740350 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0421 03:01:23.740351 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0421 03:01:23.740353 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0421 03:01:23.740154 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0421 03:01:23.740178 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0421 03:01:23.740401 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0421 03:01:23.740410 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0421 03:01:23.740171 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0421 03:01:23.740451 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0421 03:01:23.740190 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0421 03:01:23.740500 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0421 03:01:23.740209 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0421 03:01:23.740534 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0421 03:01:23.740541 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0421 03:01:23.740547 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0421 03:01:23.740551 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0421 03:01:23.739999 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0421 03:01:23.740556 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0421 03:01:23.740562 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0421 03:01:23.740564 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0421 03:01:23.740538 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0421 03:01:23.740576 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0421 03:01:23.740580 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0421 03:01:23.740588 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0421 03:01:23.740590 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0421 03:01:23.740592 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0421 03:01:23.740156 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0421 03:01:23.740595 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0421 03:01:23.740612 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0421 03:01:23.740612 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0421 03:01:23.740625 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0421 03:01:23.740632 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0421 03:01:23.740635 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0421 03:01:23.740644 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0421 03:01:23.787038 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0421 03:01:23.787056 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0421 03:01:23.787063 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0421 03:01:23.787069 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0421 03:01:23.787074 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0421 03:01:23.787079 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0421 03:01:23.787084 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0421 03:01:23.787089 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0421 03:01:23.787093 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0421 03:01:23.825744 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0421 03:01:23.825765 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0421 03:01:23.825771 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0421 03:01:23.825778 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0421 03:01:23.825782 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0421 03:01:23.825788 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0421 03:01:23.863663 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0421 03:01:23.863679 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0421 03:01:23.863694 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0421 03:01:23.863702 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0421 03:01:23.863710 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0421 03:01:23.863716 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0421 03:01:23.863721 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0421 03:01:23.863727 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0421 03:01:23.863732 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0421 03:01:23.863737 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0421 03:01:23.863742 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0421 03:01:23.863748 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0421 03:01:23.863754 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0421 03:01:23.863760 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0421 03:01:23.863768 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0421 03:01:23.863774 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0421 03:01:23.863780 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0421 03:01:24.776661 3808 default_network_controller.go:1025] Node "ip-10-0-1-71.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:25.312751 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-88.ec2.internal I0421 03:01:25.312783 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-88.ec2.internal: 39.354µs I0421 03:01:25.343882 3808 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5677c486c6-vtxhf] creating logical port openshift-image-registry_image-registry-5677c486c6-vtxhf for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:25.344015 3808 kube.go:256] Updating pod openshift-image-registry/image-registry-5677c486c6-vtxhf I0421 03:01:25.365831 3808 pod.go:62] [openshift-image-registry/image-registry-5677c486c6-vtxhf] pod update took 21.849775ms I0421 03:01:25.365855 3808 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-5677c486c6-vtxhf] addLogicalPort annotation time took 21.875993ms I0421 03:01:25.366989 3808 pods.go:271] [openshift-image-registry/image-registry-5677c486c6-vtxhf] addLogicalPort took 23.126073ms, libovsdb time 788.384µs I0421 03:01:25.379806 3808 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-qws8n] creating logical port openshift-dns_dns-default-qws8n for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:25.379930 3808 kube.go:256] Updating pod openshift-dns/dns-default-qws8n I0421 03:01:25.382820 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29612340-8dq7l for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:25.382923 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l I0421 03:01:25.383039 3808 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-8lm4x for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:25.383130 3808 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x I0421 03:01:25.397985 3808 pod.go:62] [openshift-dns/dns-default-qws8n] pod update took 18.090052ms I0421 03:01:25.398004 3808 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-qws8n] addLogicalPort annotation time took 18.112391ms I0421 03:01:25.399026 3808 pods.go:271] [openshift-dns/dns-default-qws8n] addLogicalPort took 19.237093ms, libovsdb time 758.113µs I0421 03:01:25.399384 3808 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-bghw2] creating logical port openshift-ingress-canary_ingress-canary-bghw2 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:01:25.399467 3808 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-bghw2 I0421 03:01:25.401592 3808 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x] pod update took 18.480172ms I0421 03:01:25.401613 3808 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x] addLogicalPort annotation time took 18.502331ms I0421 03:01:25.401881 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l] pod update took 18.974501ms I0421 03:01:25.401899 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l] addLogicalPort annotation time took 18.994452ms I0421 03:01:25.402599 3808 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x] addLogicalPort took 19.590592ms, libovsdb time 711.704µs I0421 03:01:25.402933 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l] addLogicalPort took 20.126902ms, libovsdb time 823.825µs I0421 03:01:25.411771 3808 pod.go:62] [openshift-ingress-canary/ingress-canary-bghw2] pod update took 12.327874ms I0421 03:01:25.411791 3808 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-bghw2] addLogicalPort annotation time took 12.362298ms I0421 03:01:25.412766 3808 pods.go:271] [openshift-ingress-canary/ingress-canary-bghw2] addLogicalPort took 13.39057ms, libovsdb time 738.384µs I0421 03:01:25.859383 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key ] I0421 03:01:25.867174 3808 cni.go:452] [openshift-ingress-canary/ingress-canary-bghw2 a31964a4124f92e46e471fcb1495b2ba0f52f525831c1dc78f31ed752a3f6f3c network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-bghw2 a31964a4124f92e46e471fcb1495b2ba0f52f525831c1dc78f31ed752a3f6f3c network default NAD default NAD key ] I0421 03:01:25.882269 3808 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x 69be1ab37cb06fe18d5f74c7fa27a29ca090bd4a5ada236efdda34e81ef42711 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x 69be1ab37cb06fe18d5f74c7fa27a29ca090bd4a5ada236efdda34e81ef42711 network default NAD default NAD key ] I0421 03:01:25.913284 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29612340-8dq7l, hostIfaceName: 7249f2745ddf06b, network: default, NAD default, SandboxID: "7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281", PCI device ID: , UID: "e70075ad-0cc2-4e9e-9ddc-97ae023de339", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0421 03:01:25.923791 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-8lm4x, hostIfaceName: 69be1ab37cb06fe, network: default, NAD default, SandboxID: "69be1ab37cb06fe18d5f74c7fa27a29ca090bd4a5ada236efdda34e81ef42711", PCI device ID: , UID: "af54892b-13b3-4a9c-ad34-36b5653f8c4d", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0421 03:01:25.928972 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-bghw2, hostIfaceName: a31964a4124f92e, network: default, NAD default, SandboxID: "a31964a4124f92e46e471fcb1495b2ba0f52f525831c1dc78f31ed752a3f6f3c", PCI device ID: , UID: "0764f5cd-b503-4c2e-81fd-c5a442476f18", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0421 03:01:25.969275 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:f5:73:db:c4:76\",\"name\":\"7249f2745ddf06b\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c44dc855-4126-4d17-a36c-84d9d9e0919a\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:01:25.974787 3808 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x 69be1ab37cb06fe18d5f74c7fa27a29ca090bd4a5ada236efdda34e81ef42711 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8lm4x 69be1ab37cb06fe18d5f74c7fa27a29ca090bd4a5ada236efdda34e81ef42711 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:83:4f:4a:32:eb\",\"name\":\"69be1ab37cb06fe\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/125eceb9-b0e7-4736-8a1e-17d15b529429\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:01:25.980678 3808 cni.go:473] [openshift-ingress-canary/ingress-canary-bghw2 a31964a4124f92e46e471fcb1495b2ba0f52f525831c1dc78f31ed752a3f6f3c network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-bghw2 a31964a4124f92e46e471fcb1495b2ba0f52f525831c1dc78f31ed752a3f6f3c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:57:80:45:7e:be\",\"name\":\"a31964a4124f92e\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35a345f7-75ce-4797-adc4-6638cb757800\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:01:26.018108 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-186.ec2.internal in network "default" I0421 03:01:26.018171 3808 node_tracker.go:235] Node ip-10-0-0-186.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-186.ec2.internal" I0421 03:01:26.018184 3808 default_network_controller.go:1025] Node "ip-10-0-0-186.ec2.internal" in remote zone "ip-10-0-0-186.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:26.018191 3808 node_tracker.go:169] Node ip-10-0-0-186.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:26.018206 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:26.104426 3808 default_network_controller.go:1025] Node "ip-10-0-0-186.ec2.internal" in remote zone "ip-10-0-0-186.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:26.140395 3808 default_network_controller.go:1025] Node "ip-10-0-0-186.ec2.internal" in remote zone "ip-10-0-0-186.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:26.140415 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-186.ec2.internal in network "default" I0421 03:01:26.140469 3808 node_tracker.go:169] Node ip-10-0-0-186.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:26.140484 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:26.141367 3808 zone_ic_handler.go:195] Time taken to create transit switch: 287.014µs I0421 03:01:26.141381 3808 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-186.ec2.internal for the network default I0421 03:01:26.142369 3808 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-186.ec2.internal" on network "default" took: 1.300211ms I0421 03:01:27.637956 3808 cni.go:452] [openshift-network-diagnostics/network-check-target-xl42b 721deb550136345032b412db34dcb25190fe84ef3c99951c6ebd04cf9376c729 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-xl42b 721deb550136345032b412db34dcb25190fe84ef3c99951c6ebd04cf9376c729 network default NAD default NAD key ] I0421 03:01:27.647500 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-xl42b, hostIfaceName: 721deb550136345, network: default, NAD default, SandboxID: "721deb550136345032b412db34dcb25190fe84ef3c99951c6ebd04cf9376c729", PCI device ID: , UID: "7acfe779-1185-4602-af4f-8fe30963c459", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0421 03:01:27.692408 3808 cni.go:473] [openshift-network-diagnostics/network-check-target-xl42b 721deb550136345032b412db34dcb25190fe84ef3c99951c6ebd04cf9376c729 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-xl42b 721deb550136345032b412db34dcb25190fe84ef3c99951c6ebd04cf9376c729 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:0e:ec:fd:10:15\",\"name\":\"721deb550136345\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed90eab8-8a8e-425f-9f72-5f66ac78348a\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:01:27.726728 3808 egressfirewall.go:723] Syncing node "ip-10-0-0-186.ec2.internal" for egress firewall I0421 03:01:27.726745 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-186.ec2.internal in network "default" I0421 03:01:27.726776 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-186.ec2.internal I0421 03:01:27.726791 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-186.ec2.internal: 21.26µs I0421 03:01:27.726799 3808 node_tracker.go:169] Node ip-10-0-0-186.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:27.726812 3808 services_controller.go:574] Full service sync requested for network=default I0421 03:01:27.726837 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-186.ec2.internal", IP: 10.0.0.186 to PMTUD blocking rules I0421 03:01:27.726852 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-186.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0421 03:01:27.733113 3808 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-186.ec2.internal. Updating no re-route policies I0421 03:01:27.741923 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-186.ec2.internal", IP: 10.0.0.186 to PMTUD blocking rules I0421 03:01:27.741941 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-186.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0421 03:01:31.690772 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-186.ec2.internal I0421 03:01:31.690807 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-186.ec2.internal: 44.265µs I0421 03:01:32.422696 3808 egressfirewall.go:723] Syncing node "ip-10-0-1-71.ec2.internal" for egress firewall I0421 03:01:32.422719 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0421 03:01:32.422752 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 35.354µs I0421 03:01:32.422783 3808 default_network_controller.go:1025] Node "ip-10-0-1-71.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0421 03:01:34.624175 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0421 03:01:35.630752 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0421 03:01:37.339954 3808 default_network_controller.go:1025] Node "ip-10-0-1-71.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:38.661567 3808 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-c8pkv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:01:38.661611 3808 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-c8pkv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:01:39.569877 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0421 03:01:39.569912 3808 node_tracker.go:235] Node ip-10-0-1-71.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-71.ec2.internal" I0421 03:01:39.569914 3808 default_network_controller.go:1025] Node "ip-10-0-1-71.ec2.internal" in remote zone "ip-10-0-1-71.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:39.569930 3808 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:39.569944 3808 services_controller.go:574] Full service sync requested for network=default W0421 03:01:39.570018 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0421 03:01:39.570397 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0421 03:01:39.657277 3808 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-c8pkv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:01:39.657294 3808 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-c8pkv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:01:39.716926 3808 default_network_controller.go:1025] Node "ip-10-0-1-71.ec2.internal" in remote zone "ip-10-0-1-71.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0421 03:01:39.716931 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0421 03:01:39.716982 3808 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:39.716996 3808 services_controller.go:574] Full service sync requested for network=default W0421 03:01:39.717385 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0421 03:01:39.717514 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0421 03:01:39.717884 3808 zone_ic_handler.go:195] Time taken to create transit switch: 262.385µs I0421 03:01:39.717898 3808 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-71.ec2.internal for the network default I0421 03:01:39.718995 3808 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-71.ec2.internal" on network "default" took: 1.384065ms W0421 03:01:40.043898 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0421 03:01:40.276389 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0421 03:01:40.276398 3808 egressfirewall.go:723] Syncing node "ip-10-0-1-71.ec2.internal" for egress firewall I0421 03:01:40.276419 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0421 03:01:40.276423 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-71.ec2.internal", IP: 10.0.1.71 to PMTUD blocking rules I0421 03:01:40.276431 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 18.033µs I0421 03:01:40.276433 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-71.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0421 03:01:40.276445 3808 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0421 03:01:40.276458 3808 services_controller.go:574] Full service sync requested for network=default W0421 03:01:40.276538 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0421 03:01:40.276771 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0421 03:01:40.283321 3808 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-71.ec2.internal. Updating no re-route policies I0421 03:01:40.293883 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-71.ec2.internal", IP: 10.0.1.71 to PMTUD blocking rules I0421 03:01:40.293898 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-71.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0421 03:01:44.191693 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0421 03:01:44.191721 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 38.482µs W0421 03:01:47.655184 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0421 03:01:52.596419 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0421 03:02:05.634431 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0421 03:03:30.316554 3808 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5dc7f9d99-ntbcv] creating logical port openshift-ingress_router-default-5dc7f9d99-ntbcv for pod on switch ip-10-0-2-88.ec2.internal I0421 03:03:30.316641 3808 kube.go:256] Updating pod openshift-ingress/router-default-5dc7f9d99-ntbcv I0421 03:03:30.321448 3808 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5b58d99ffc-4kdzg] creating logical port openshift-ingress_router-default-5b58d99ffc-4kdzg for pod on switch ip-10-0-2-88.ec2.internal I0421 03:03:30.321514 3808 kube.go:256] Updating pod openshift-ingress/router-default-5b58d99ffc-4kdzg I0421 03:03:30.342871 3808 pod.go:62] [openshift-ingress/router-default-5dc7f9d99-ntbcv] pod update took 26.255344ms I0421 03:03:30.342893 3808 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5dc7f9d99-ntbcv] addLogicalPort annotation time took 26.280418ms I0421 03:03:30.343829 3808 pods.go:271] [openshift-ingress/router-default-5dc7f9d99-ntbcv] addLogicalPort took 27.289628ms, libovsdb time 710.462µs I0421 03:03:30.352199 3808 pod.go:62] [openshift-ingress/router-default-5b58d99ffc-4kdzg] pod update took 30.706053ms I0421 03:03:30.352216 3808 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5b58d99ffc-4kdzg] addLogicalPort annotation time took 30.725704ms I0421 03:03:30.353043 3808 pods.go:271] [openshift-ingress/router-default-5b58d99ffc-4kdzg] addLogicalPort took 31.60464ms, libovsdb time 613.851µs W0421 03:03:33.822587 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0421 03:03:35.175887 3808 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0421 03:03:35.176816 3808 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 907.265µs I0421 03:03:35.207471 3808 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0421 03:03:35.896619 3808 namespace.go:98] [openshift-service-ca] adding namespace I0421 03:03:35.897412 3808 namespace.go:102] [openshift-service-ca] adding namespace took 773.717µs I0421 03:03:35.919284 3808 namespace.go:142] [openshift-service-ca] updating namespace W0421 03:03:44.808233 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0421 03:03:59.387888 3808 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0421 03:03:59.462005 3808 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-r6298] creating logical port openshift-insights_insights-runtime-extractor-r6298 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:03:59.462098 3808 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-r6298 I0421 03:03:59.482557 3808 pod.go:62] [openshift-insights/insights-runtime-extractor-r6298] pod update took 20.4877ms I0421 03:03:59.482585 3808 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-r6298] addLogicalPort annotation time took 20.519949ms I0421 03:03:59.483638 3808 pods.go:271] [openshift-insights/insights-runtime-extractor-r6298] addLogicalPort took 21.650553ms, libovsdb time 793.326µs I0421 03:03:59.825639 3808 cni.go:452] [openshift-insights/insights-runtime-extractor-r6298 9c576bb86a36605c7e480ba392400196c117287c9ee0966380df8a1c2534932f network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-r6298 9c576bb86a36605c7e480ba392400196c117287c9ee0966380df8a1c2534932f network default NAD default NAD key ] I0421 03:03:59.835688 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-r6298, hostIfaceName: 9c576bb86a36605, network: default, NAD default, SandboxID: "9c576bb86a36605c7e480ba392400196c117287c9ee0966380df8a1c2534932f", PCI device ID: , UID: "ac46f6c9-4a84-48fb-a815-c3cd1db3b07b", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0421 03:03:59.881201 3808 cni.go:473] [openshift-insights/insights-runtime-extractor-r6298 9c576bb86a36605c7e480ba392400196c117287c9ee0966380df8a1c2534932f network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-r6298 9c576bb86a36605c7e480ba392400196c117287c9ee0966380df8a1c2534932f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:33:f4:a6:42:ea\",\"name\":\"9c576bb86a36605\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a94b8373-1d38-471d-984c-6e0d775979fe\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:04:00.997726 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5677c486c6-vtxhf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:00.997769 3808 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5677c486c6-vtxhf I0421 03:04:00.999090 3808 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5677c486c6-vtxhf, ips: 10.128.0.5 I0421 03:04:00.999117 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5677c486c6-vtxhf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:01.008707 3808 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c954bbfc-cl4s7] creating logical port openshift-image-registry_image-registry-7c954bbfc-cl4s7 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:01.008803 3808 kube.go:256] Updating pod openshift-image-registry/image-registry-7c954bbfc-cl4s7 I0421 03:04:01.016767 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5677c486c6-vtxhf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:01.016788 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5677c486c6-vtxhf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:01.021192 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5677c486c6-vtxhf I0421 03:04:01.021212 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5677c486c6-vtxhf I0421 03:04:01.028418 3808 pod.go:62] [openshift-image-registry/image-registry-7c954bbfc-cl4s7] pod update took 19.646624ms I0421 03:04:01.028439 3808 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7c954bbfc-cl4s7] addLogicalPort annotation time took 19.671119ms I0421 03:04:01.029532 3808 pods.go:271] [openshift-image-registry/image-registry-7c954bbfc-cl4s7] addLogicalPort took 20.839244ms, libovsdb time 834.107µs I0421 03:04:01.363279 3808 cni.go:452] [openshift-image-registry/image-registry-7c954bbfc-cl4s7 0aa4e87575d4dd571b42a1c08be1b1540d731215ccba3ec427e0f469bd3d277e network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7c954bbfc-cl4s7 0aa4e87575d4dd571b42a1c08be1b1540d731215ccba3ec427e0f469bd3d277e network default NAD default NAD key ] I0421 03:04:01.373420 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7c954bbfc-cl4s7, hostIfaceName: 0aa4e87575d4dd5, network: default, NAD default, SandboxID: "0aa4e87575d4dd571b42a1c08be1b1540d731215ccba3ec427e0f469bd3d277e", PCI device ID: , UID: "35150a60-a5c1-41e6-8cfc-f66e4e3b5224", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0421 03:04:01.396934 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-748564bdc9-j47ll of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:01.397464 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-748564bdc9-j47ll of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:01.412386 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-748564bdc9-j47ll of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:01.412403 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-748564bdc9-j47ll of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:01.417458 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-748564bdc9-j47ll I0421 03:04:01.417480 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-748564bdc9-j47ll I0421 03:04:01.418412 3808 cni.go:473] [openshift-image-registry/image-registry-7c954bbfc-cl4s7 0aa4e87575d4dd571b42a1c08be1b1540d731215ccba3ec427e0f469bd3d277e network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7c954bbfc-cl4s7 0aa4e87575d4dd571b42a1c08be1b1540d731215ccba3ec427e0f469bd3d277e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:bf:82:e3:e9:87\",\"name\":\"0aa4e87575d4dd5\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c918a018-8066-4ab2-b1e9-db284e2cd292\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:04:01.921134 3808 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0421 03:04:01.921171 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0421 03:04:01.921177 3808 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0421 03:04:01.995168 3808 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0421 03:04:02.401456 3808 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0421 03:04:02.484398 3808 cni.go:452] [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key ] I0421 03:04:02.487793 3808 cni.go:452] [openshift-ingress/router-default-5dc7f9d99-ntbcv ae89ec5bf436596a220e97623764dc3b1cadc6f0981147bcc21431f4a7afc388 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5dc7f9d99-ntbcv ae89ec5bf436596a220e97623764dc3b1cadc6f0981147bcc21431f4a7afc388 network default NAD default NAD key ] I0421 03:04:02.495473 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5b58d99ffc-4kdzg, hostIfaceName: a38dd5f1e14be99, network: default, NAD default, SandboxID: "a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e", PCI device ID: , UID: "0ab12948-e962-45f5-867d-c2751e7baa72", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0421 03:04:02.499173 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5dc7f9d99-ntbcv, hostIfaceName: ae89ec5bf436596, network: default, NAD default, SandboxID: "ae89ec5bf436596a220e97623764dc3b1cadc6f0981147bcc21431f4a7afc388", PCI device ID: , UID: "9cd5894b-a450-45d2-b35d-a24b2c5b467c", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0421 03:04:02.543850 3808 cni.go:473] [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:4f:08:81:a5:90\",\"name\":\"a38dd5f1e14be99\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10f5da35-3243-4eb2-82be-d62e05995cd9\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:04:02.561129 3808 cni.go:473] [openshift-ingress/router-default-5dc7f9d99-ntbcv ae89ec5bf436596a220e97623764dc3b1cadc6f0981147bcc21431f4a7afc388 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5dc7f9d99-ntbcv ae89ec5bf436596a220e97623764dc3b1cadc6f0981147bcc21431f4a7afc388 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:ec:ca:33:b4:57\",\"name\":\"ae89ec5bf436596\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f67368a3-c461-477c-b0d9-84341f956d86\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:04:03.000058 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:03.000076 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:03.001103 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.001119 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.001124 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.055517 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:03.055533 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:03.055606 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.055620 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.055626 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.408944 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.408960 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.408966 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:03.409046 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:03.409055 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:04.009745 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0421 03:04:04.009774 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default I0421 03:04:04.093979 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0421 03:04:04.107109 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:04.107129 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0421 03:04:04.107155 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default I0421 03:04:04.203964 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0421 03:04:05.277664 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-jm5jm for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:05.277760 3808 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm I0421 03:04:05.299262 3808 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm] pod update took 21.526138ms I0421 03:04:05.299287 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm] addLogicalPort annotation time took 21.555359ms I0421 03:04:05.300277 3808 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm] addLogicalPort took 22.63194ms, libovsdb time 732.23µs I0421 03:04:05.633904 3808 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm 5a69503eb8ccf6bd213eb36b3de8ae41470674563400279043ceec0ce69cb5de network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm 5a69503eb8ccf6bd213eb36b3de8ae41470674563400279043ceec0ce69cb5de network default NAD default NAD key ] I0421 03:04:05.644590 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-jm5jm, hostIfaceName: 5a69503eb8ccf6b, network: default, NAD default, SandboxID: "5a69503eb8ccf6bd213eb36b3de8ae41470674563400279043ceec0ce69cb5de", PCI device ID: , UID: "f0594262-266a-4b47-93c2-7bc2053f11a4", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0421 03:04:05.691839 3808 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm 5a69503eb8ccf6bd213eb36b3de8ae41470674563400279043ceec0ce69cb5de network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-jm5jm 5a69503eb8ccf6bd213eb36b3de8ae41470674563400279043ceec0ce69cb5de network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:3e:4a:8f:78:b1\",\"name\":\"5a69503eb8ccf6b\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8a35f0a5-4ef0-45c3-959a-59aa1cb4acad\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:04:07.014017 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0421 03:04:07.036462 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0421 03:04:13.907304 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0421 03:04:13.907332 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0421 03:04:13.911676 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:13.911788 3808 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0421 03:04:13.938870 3808 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 27.130456ms I0421 03:04:13.938895 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 27.157311ms I0421 03:04:13.940097 3808 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 28.429809ms, libovsdb time 898.401µs I0421 03:04:14.867786 3808 cni.go:452] [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key ] I0421 03:04:14.882010 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 22df69ab469b5bc, network: default, NAD default, SandboxID: "22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e", PCI device ID: , UID: "0fab19f3-5780-487c-a7d5-073c3f72da48", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0421 03:04:14.933536 3808 cni.go:473] [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:b1:84:05:62:cf\",\"name\":\"22df69ab469b5bc\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92bad5a9-f6f8-4282-8ceb-a8c34db133b6\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:04:16.538031 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:16.538058 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:16.538066 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:17.068869 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:17.068888 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:17.068894 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0421 03:04:17.441678 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-77999f96f6-7zjww] creating logical port openshift-monitoring_metrics-server-77999f96f6-7zjww for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:17.441848 3808 kube.go:256] Updating pod openshift-monitoring/metrics-server-77999f96f6-7zjww I0421 03:04:17.462853 3808 pod.go:62] [openshift-monitoring/metrics-server-77999f96f6-7zjww] pod update took 21.037532ms I0421 03:04:17.462877 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-77999f96f6-7zjww] addLogicalPort annotation time took 21.067149ms I0421 03:04:17.464199 3808 pods.go:271] [openshift-monitoring/metrics-server-77999f96f6-7zjww] addLogicalPort took 22.541379ms, libovsdb time 976.987µs I0421 03:04:17.555174 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-fq5rm for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:17.555274 3808 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm I0421 03:04:17.574465 3808 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm] pod update took 19.217262ms I0421 03:04:17.574496 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm] addLogicalPort annotation time took 19.252136ms I0421 03:04:17.575746 3808 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm] addLogicalPort took 20.605466ms, libovsdb time 870.635µs I0421 03:04:18.100559 3808 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm 2ac4a9b94ca1ec7bf8ea80bc3be28e7e68f7375e708fc95700d3ee67e77d4395 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm 2ac4a9b94ca1ec7bf8ea80bc3be28e7e68f7375e708fc95700d3ee67e77d4395 network default NAD default NAD key ] I0421 03:04:18.100559 3808 cni.go:452] [openshift-monitoring/metrics-server-77999f96f6-7zjww 88a2441c5b6fa60cfb255271735e678639bd61038b4ea47a7c22ecdbcfc6e100 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-77999f96f6-7zjww 88a2441c5b6fa60cfb255271735e678639bd61038b4ea47a7c22ecdbcfc6e100 network default NAD default NAD key ] I0421 03:04:18.112283 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-77999f96f6-7zjww, hostIfaceName: 88a2441c5b6fa60, network: default, NAD default, SandboxID: "88a2441c5b6fa60cfb255271735e678639bd61038b4ea47a7c22ecdbcfc6e100", PCI device ID: , UID: "e1a93b1c-4adf-49ba-99b0-e97404d755dc", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0421 03:04:18.112921 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-fq5rm, hostIfaceName: 2ac4a9b94ca1ec7, network: default, NAD default, SandboxID: "2ac4a9b94ca1ec7bf8ea80bc3be28e7e68f7375e708fc95700d3ee67e77d4395", PCI device ID: , UID: "6996cc32-f950-4cc7-a417-f251466c91a5", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0421 03:04:18.163086 3808 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm 2ac4a9b94ca1ec7bf8ea80bc3be28e7e68f7375e708fc95700d3ee67e77d4395 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-fq5rm 2ac4a9b94ca1ec7bf8ea80bc3be28e7e68f7375e708fc95700d3ee67e77d4395 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:b9:00:3a:72:dc\",\"name\":\"2ac4a9b94ca1ec7\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5fb3a682-b755-4e76-8932-0d1968a8ae4d\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:04:18.176972 3808 cni.go:473] [openshift-monitoring/metrics-server-77999f96f6-7zjww 88a2441c5b6fa60cfb255271735e678639bd61038b4ea47a7c22ecdbcfc6e100 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-77999f96f6-7zjww 88a2441c5b6fa60cfb255271735e678639bd61038b4ea47a7c22ecdbcfc6e100 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:44:a4:ff:86:c7\",\"name\":\"88a2441c5b6fa60\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9b28dae8-9c1f-4bbf-b1e1-373570867a24\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:04:18.181650 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-6774f85884-t6z5h] creating logical port openshift-monitoring_telemeter-client-6774f85884-t6z5h for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:18.181767 3808 kube.go:256] Updating pod openshift-monitoring/telemeter-client-6774f85884-t6z5h I0421 03:04:18.202471 3808 pod.go:62] [openshift-monitoring/telemeter-client-6774f85884-t6z5h] pod update took 20.729878ms I0421 03:04:18.202502 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-6774f85884-t6z5h] addLogicalPort annotation time took 20.763771ms I0421 03:04:18.204007 3808 pods.go:271] [openshift-monitoring/telemeter-client-6774f85884-t6z5h] addLogicalPort took 22.368616ms, libovsdb time 1.14256ms I0421 03:04:18.537214 3808 cni.go:452] [openshift-monitoring/telemeter-client-6774f85884-t6z5h af862a608285ea15fdddf53a75fa18a4b291fa9ebbabe88c2c55b55e0289758a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-6774f85884-t6z5h af862a608285ea15fdddf53a75fa18a4b291fa9ebbabe88c2c55b55e0289758a network default NAD default NAD key ] I0421 03:04:18.547436 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-6774f85884-t6z5h, hostIfaceName: af862a608285ea1, network: default, NAD default, SandboxID: "af862a608285ea15fdddf53a75fa18a4b291fa9ebbabe88c2c55b55e0289758a", PCI device ID: , UID: "64625f05-c5f1-4fdd-bc77-0ae2a69e541c", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0421 03:04:18.592794 3808 cni.go:473] [openshift-monitoring/telemeter-client-6774f85884-t6z5h af862a608285ea15fdddf53a75fa18a4b291fa9ebbabe88c2c55b55e0289758a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-6774f85884-t6z5h af862a608285ea15fdddf53a75fa18a4b291fa9ebbabe88c2c55b55e0289758a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:17:2f:dc:b0:73\",\"name\":\"af862a608285ea1\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4507f5f3-d80f-42c2-a20a-54c2f8e24356\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:04:20.534223 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0421 03:04:20.534248 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0421 03:04:21.032995 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0421 03:04:21.081350 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:21.081374 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:21.081382 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:21.081388 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:21.088396 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0421 03:04:21.116713 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0421 03:04:21.138214 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0421 03:04:21.138236 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0421 03:04:21.151342 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin I0421 03:04:21.688437 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:21.689039 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:21.689074 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:04:21.689086 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0421 03:04:21.689099 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:04:21.689103 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0421 03:04:22.050228 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0421 03:04:22.050249 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0421 03:04:22.549126 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:22.549173 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:22.549181 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0421 03:04:22.549185 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0421 03:04:22.717526 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:22.717543 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:22.717558 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:22.717566 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s9rgp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0421 03:04:23.014042 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0421 03:04:24.284739 3808 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-vt8mh] creating logical port openshift-console_downloads-5c54f4944f-vt8mh for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:24.284838 3808 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-vt8mh I0421 03:04:24.301266 3808 pod.go:62] [openshift-console/downloads-5c54f4944f-vt8mh] pod update took 16.45474ms I0421 03:04:24.301291 3808 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-vt8mh] addLogicalPort annotation time took 16.483378ms I0421 03:04:24.302571 3808 pods.go:271] [openshift-console/downloads-5c54f4944f-vt8mh] addLogicalPort took 17.851526ms, libovsdb time 893.272µs I0421 03:04:24.642858 3808 cni.go:452] [openshift-console/downloads-5c54f4944f-vt8mh 6227398df0fcec8e4daeef531983a7359171a415eeda3f366db11f55e2d78197 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-vt8mh 6227398df0fcec8e4daeef531983a7359171a415eeda3f366db11f55e2d78197 network default NAD default NAD key ] I0421 03:04:24.652557 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-vt8mh, hostIfaceName: 6227398df0fcec8, network: default, NAD default, SandboxID: "6227398df0fcec8e4daeef531983a7359171a415eeda3f366db11f55e2d78197", PCI device ID: , UID: "315cee46-ea63-40c8-bdff-8c67dc73b4ab", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0421 03:04:24.699352 3808 cni.go:473] [openshift-console/downloads-5c54f4944f-vt8mh 6227398df0fcec8e4daeef531983a7359171a415eeda3f366db11f55e2d78197 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-vt8mh 6227398df0fcec8e4daeef531983a7359171a415eeda3f366db11f55e2d78197 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:73:76:e9:68:66\",\"name\":\"6227398df0fcec8\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3151bc22-7f7b-454b-a47d-3f45623ed965\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:04:27.679349 3808 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0421 03:04:27.688965 3808 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0421 03:04:32.308276 3808 base_network_controller_pods.go:487] [default/openshift-console/console-58f6467f44-ph6pc] creating logical port openshift-console_console-58f6467f44-ph6pc for pod on switch ip-10-0-2-88.ec2.internal I0421 03:04:32.308373 3808 kube.go:256] Updating pod openshift-console/console-58f6467f44-ph6pc I0421 03:04:32.330642 3808 pod.go:62] [openshift-console/console-58f6467f44-ph6pc] pod update took 22.295319ms I0421 03:04:32.330663 3808 base_network_controller_pods.go:951] [default/openshift-console/console-58f6467f44-ph6pc] addLogicalPort annotation time took 22.322597ms I0421 03:04:32.331948 3808 pods.go:271] [openshift-console/console-58f6467f44-ph6pc] addLogicalPort took 23.686978ms, libovsdb time 988.58µs I0421 03:04:32.677677 3808 cni.go:452] [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key ] I0421 03:04:32.687689 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-58f6467f44-ph6pc, hostIfaceName: 86e92ccd821b167, network: default, NAD default, SandboxID: "86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1", PCI device ID: , UID: "8065314b-0f25-42c3-beab-a40169be8edf", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0421 03:04:32.737280 3808 cni.go:473] [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:9a:67:3d:02:fb\",\"name\":\"86e92ccd821b167\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ceae455-f312-4760-acd8-8f72a700e2b6\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:04:33.153981 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0421 03:04:37.138893 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0421 03:04:37.159427 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0421 03:04:41.202948 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0421 03:04:42.047084 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0421 03:04:42.047112 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0421 03:04:42.101809 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0421 03:04:43.141362 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0421 03:04:43.141442 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0421 03:04:43.175554 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0421 03:04:43.201893 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0421 03:04:43.616374 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0421 03:04:44.923317 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:44.923344 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:04:44.923353 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0421 03:04:48.346126 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-748564bdc9-fzm5j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:48.346776 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-748564bdc9-fzm5j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:04:49.151208 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-748564bdc9-fzm5j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:49.151229 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-748564bdc9-fzm5j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:04:49.157819 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-748564bdc9-fzm5j I0421 03:04:49.157847 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-748564bdc9-fzm5j W0421 03:04:51.665334 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0421 03:04:51.665356 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0421 03:04:51.703267 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0421 03:04:51.703290 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0421 03:04:55.672523 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0421 03:04:59.202485 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0421 03:05:05.451370 3808 cni.go:452] [openshift-multus/network-metrics-daemon-vspcz bb13712c3c5b7b6b72af45330423c979b03338943b4042061c5364c6c61dcb8f network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-vspcz bb13712c3c5b7b6b72af45330423c979b03338943b4042061c5364c6c61dcb8f network default NAD default NAD key ] I0421 03:05:05.460661 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-vspcz, hostIfaceName: bb13712c3c5b7b6, network: default, NAD default, SandboxID: "bb13712c3c5b7b6b72af45330423c979b03338943b4042061c5364c6c61dcb8f", PCI device ID: , UID: "086dd386-d9a0-4d9a-8be6-99c165535a9b", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0421 03:05:05.512256 3808 cni.go:473] [openshift-multus/network-metrics-daemon-vspcz bb13712c3c5b7b6b72af45330423c979b03338943b4042061c5364c6c61dcb8f network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-vspcz bb13712c3c5b7b6b72af45330423c979b03338943b4042061c5364c6c61dcb8f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:97:49:ed:1a:ba\",\"name\":\"bb13712c3c5b7b6\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1ca0db60-81c2-4101-811d-c654171b7a44\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:05:06.387471 3808 cni.go:452] [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key ] I0421 03:05:06.471026 3808 cni.go:473] [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-58f6467f44-ph6pc 86e92ccd821b16722011c2a3ab1c627d3ce1456a940ba7df299275adad3879e1 network default NAD default NAD key default], result "{}", err I0421 03:05:06.539268 3808 obj_retry.go:498] Detected object openshift-console/console-58f6467f44-ph6pc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:06.539307 3808 pods.go:176] Deleting pod: openshift-console/console-58f6467f44-ph6pc I0421 03:05:06.540579 3808 pods.go:236] Attempting to release IPs for pod: openshift-console/console-58f6467f44-ph6pc, ips: 10.128.0.20 I0421 03:05:06.540613 3808 obj_retry.go:498] Detected object openshift-console/console-58f6467f44-ph6pc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:06.554507 3808 base_network_controller_pods.go:487] [default/openshift-console/console-6fcf9ffcd-qqv8s] creating logical port openshift-console_console-6fcf9ffcd-qqv8s for pod on switch ip-10-0-2-88.ec2.internal I0421 03:05:06.554640 3808 kube.go:256] Updating pod openshift-console/console-6fcf9ffcd-qqv8s I0421 03:05:06.574869 3808 pod.go:62] [openshift-console/console-6fcf9ffcd-qqv8s] pod update took 20.292504ms I0421 03:05:06.574898 3808 base_network_controller_pods.go:951] [default/openshift-console/console-6fcf9ffcd-qqv8s] addLogicalPort annotation time took 20.325899ms I0421 03:05:06.576083 3808 pods.go:271] [openshift-console/console-6fcf9ffcd-qqv8s] addLogicalPort took 21.595007ms, libovsdb time 820.036µs I0421 03:05:06.942327 3808 cni.go:452] [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key ] I0421 03:05:06.952169 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6fcf9ffcd-qqv8s, hostIfaceName: 160976fc49807f6, network: default, NAD default, SandboxID: "160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019", PCI device ID: , UID: "6095e0db-eb10-428d-945e-a2afca3d37a8", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0421 03:05:07.000245 3808 cni.go:473] [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:82:fe:0b:01:17\",\"name\":\"160976fc49807f6\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c46c9840-a704-4a3e-8bd8-c296caa6619c\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:05:07.268550 3808 obj_retry.go:492] Detected object openshift-console/console-58f6467f44-ph6pc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:07.268575 3808 obj_retry.go:492] Detected object openshift-console/console-58f6467f44-ph6pc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:07.274380 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-58f6467f44-ph6pc I0421 03:05:07.274400 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-58f6467f44-ph6pc I0421 03:05:16.988081 3808 obj_retry.go:498] Detected object openshift-console/console-58f6467f44-9h9l8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:16.988783 3808 obj_retry.go:498] Detected object openshift-console/console-58f6467f44-9h9l8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:17.268926 3808 obj_retry.go:492] Detected object openshift-console/console-58f6467f44-9h9l8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:17.268950 3808 obj_retry.go:492] Detected object openshift-console/console-58f6467f44-9h9l8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:17.275352 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-58f6467f44-9h9l8 I0421 03:05:17.275380 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-58f6467f44-9h9l8 W0421 03:05:17.324379 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0421 03:05:19.235625 3808 cni.go:452] [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key ] I0421 03:05:19.311607 3808 cni.go:473] [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-5b58d99ffc-4kdzg a38dd5f1e14be997b76380c51d1dadd14629962653c2ce78d727afc75a1b947e network default NAD default NAD key default], result "{}", err I0421 03:05:19.375104 3808 obj_retry.go:498] Detected object openshift-ingress/router-default-5b58d99ffc-4kdzg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:19.375131 3808 pods.go:176] Deleting pod: openshift-ingress/router-default-5b58d99ffc-4kdzg I0421 03:05:19.376290 3808 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5b58d99ffc-4kdzg, ips: 10.128.0.11 I0421 03:05:19.376317 3808 obj_retry.go:498] Detected object openshift-ingress/router-default-5b58d99ffc-4kdzg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:20.337540 3808 obj_retry.go:492] Detected object openshift-ingress/router-default-5b58d99ffc-4kdzg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:20.337560 3808 obj_retry.go:492] Detected object openshift-ingress/router-default-5b58d99ffc-4kdzg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:20.340099 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5b58d99ffc-4kdzg I0421 03:05:20.340122 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5b58d99ffc-4kdzg I0421 03:05:34.874772 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:34.875364 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:35.403855 3808 cni.go:452] [openshift-dns/dns-default-qws8n 00289b8c6b9be45d35188469e014ca4260ce9d02226dd7a6f322d07ea8de7a4d network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-qws8n 00289b8c6b9be45d35188469e014ca4260ce9d02226dd7a6f322d07ea8de7a4d network default NAD default NAD key ] I0421 03:05:35.413876 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-qws8n, hostIfaceName: 00289b8c6b9be45, network: default, NAD default, SandboxID: "00289b8c6b9be45d35188469e014ca4260ce9d02226dd7a6f322d07ea8de7a4d", PCI device ID: , UID: "1cf35690-37a2-487c-b558-daf72c5bc01f", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0421 03:05:35.462433 3808 cni.go:473] [openshift-dns/dns-default-qws8n 00289b8c6b9be45d35188469e014ca4260ce9d02226dd7a6f322d07ea8de7a4d network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-qws8n 00289b8c6b9be45d35188469e014ca4260ce9d02226dd7a6f322d07ea8de7a4d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:1b:0f:a1:cd:c7\",\"name\":\"00289b8c6b9be45\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ee6dc237-2921-4795-a9e6-ddda835c3746\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:05:35.852982 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:35.853005 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:35.859682 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0421 03:05:35.859705 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0421 03:05:39.161212 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:39.161800 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:39.881003 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:39.881030 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:39.888738 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0421 03:05:39.888760 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0421 03:05:47.408121 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0421 03:05:47.408156 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0421 03:05:47.408166 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0421 03:05:55.302570 3808 cni.go:452] [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key ] I0421 03:05:55.368431 3808 cni.go:473] [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6fcf9ffcd-qqv8s 160976fc49807f6338e2bd6619bec8af429b6d076a5b7ffe14f5c847e860d019 network default NAD default NAD key default], result "{}", err I0421 03:05:55.416302 3808 obj_retry.go:498] Detected object openshift-console/console-6fcf9ffcd-qqv8s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:55.416331 3808 pods.go:176] Deleting pod: openshift-console/console-6fcf9ffcd-qqv8s I0421 03:05:55.417505 3808 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6fcf9ffcd-qqv8s, ips: 10.128.0.21 I0421 03:05:55.417528 3808 obj_retry.go:498] Detected object openshift-console/console-6fcf9ffcd-qqv8s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:55.427490 3808 base_network_controller_pods.go:487] [default/openshift-console/console-7d8bbf4748-6p5qf] creating logical port openshift-console_console-7d8bbf4748-6p5qf for pod on switch ip-10-0-2-88.ec2.internal I0421 03:05:55.427618 3808 kube.go:256] Updating pod openshift-console/console-7d8bbf4748-6p5qf I0421 03:05:55.444662 3808 pod.go:62] [openshift-console/console-7d8bbf4748-6p5qf] pod update took 17.066225ms I0421 03:05:55.444684 3808 base_network_controller_pods.go:951] [default/openshift-console/console-7d8bbf4748-6p5qf] addLogicalPort annotation time took 17.091023ms I0421 03:05:55.446104 3808 pods.go:271] [openshift-console/console-7d8bbf4748-6p5qf] addLogicalPort took 18.623667ms, libovsdb time 1.016847ms I0421 03:05:55.766076 3808 obj_retry.go:492] Detected object openshift-console/console-6fcf9ffcd-qqv8s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:55.766100 3808 obj_retry.go:492] Detected object openshift-console/console-6fcf9ffcd-qqv8s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:55.771368 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6fcf9ffcd-qqv8s I0421 03:05:55.771390 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6fcf9ffcd-qqv8s I0421 03:05:55.790782 3808 cni.go:452] [openshift-console/console-7d8bbf4748-6p5qf 91bd8b7328a6e378604497c5fc06f59c6963fce6724345104b4524ac4794c025 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7d8bbf4748-6p5qf 91bd8b7328a6e378604497c5fc06f59c6963fce6724345104b4524ac4794c025 network default NAD default NAD key ] I0421 03:05:55.801386 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7d8bbf4748-6p5qf, hostIfaceName: 91bd8b7328a6e37, network: default, NAD default, SandboxID: "91bd8b7328a6e378604497c5fc06f59c6963fce6724345104b4524ac4794c025", PCI device ID: , UID: "0539d682-95d4-402e-8b76-509a07fb4bc3", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0421 03:05:55.850445 3808 cni.go:473] [openshift-console/console-7d8bbf4748-6p5qf 91bd8b7328a6e378604497c5fc06f59c6963fce6724345104b4524ac4794c025 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7d8bbf4748-6p5qf 91bd8b7328a6e378604497c5fc06f59c6963fce6724345104b4524ac4794c025 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:17:7c:8d:84:59\",\"name\":\"91bd8b7328a6e37\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bf193255-c0d9-4afd-b23e-f2955c4e8b34\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:05:57.495126 3808 cni.go:452] [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key ] I0421 03:05:57.574919 3808 cni.go:473] [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 22df69ab469b5bc60e3fd1cd1491a1197182a8549fe98bb2cca8591969e1ff6e network default NAD default NAD key default], result "{}", err I0421 03:05:57.637312 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:57.637341 3808 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0421 03:05:57.638567 3808 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.128.0.15 I0421 03:05:57.638601 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:05:58.489520 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:58.489546 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:05:58.496288 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0421 03:05:58.496322 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0421 03:05:58.531951 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:05:58.532081 3808 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0421 03:05:58.558297 3808 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 26.250828ms I0421 03:05:58.558318 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 26.27777ms I0421 03:05:58.559491 3808 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 27.556623ms, libovsdb time 874.594µs I0421 03:05:58.889569 3808 cni.go:452] [openshift-monitoring/alertmanager-main-0 e5be431989a79cdfe894d7d3c88d56ba72845fe36f9232c895883f7ab9d0a079 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 e5be431989a79cdfe894d7d3c88d56ba72845fe36f9232c895883f7ab9d0a079 network default NAD default NAD key ] I0421 03:05:58.899447 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: e5be431989a79cd, network: default, NAD default, SandboxID: "e5be431989a79cdfe894d7d3c88d56ba72845fe36f9232c895883f7ab9d0a079", PCI device ID: , UID: "b3b1a01e-c896-4fca-a2ba-d991422e80ce", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0421 03:05:58.947217 3808 cni.go:473] [openshift-monitoring/alertmanager-main-0 e5be431989a79cdfe894d7d3c88d56ba72845fe36f9232c895883f7ab9d0a079 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 e5be431989a79cdfe894d7d3c88d56ba72845fe36f9232c895883f7ab9d0a079 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:a8:fd:40:f1:4f\",\"name\":\"e5be431989a79cd\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b786a69e-195e-417d-80dc-d64a7c548b4e\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:06:06.535906 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0421 03:06:06.721331 3808 obj_retry.go:498] Detected object openshift-console/console-6fcf9ffcd-9zsft of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:06:06.721867 3808 obj_retry.go:498] Detected object openshift-console/console-6fcf9ffcd-9zsft of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:06:07.001174 3808 obj_retry.go:492] Detected object openshift-console/console-6fcf9ffcd-9zsft of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:06:07.001194 3808 obj_retry.go:492] Detected object openshift-console/console-6fcf9ffcd-9zsft of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:06:07.006782 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6fcf9ffcd-9zsft I0421 03:06:07.006802 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6fcf9ffcd-9zsft I0421 03:06:10.366603 3808 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0421 03:06:28.943059 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:06:28.943085 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0421 03:06:28.943090 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0421 03:06:42.507982 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:06:42.508534 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:06:42.921424 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:06:42.921447 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:06:42.929726 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0421 03:06:42.929748 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0421 03:07:08.262214 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0421 03:07:10.616689 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0421 03:07:22.818666 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key ] I0421 03:07:22.895189 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l 7249f2745ddf06b0e58e78d4cd39ed6ef25b5d42c5c20f48568a0438857ac281 network default NAD default NAD key default], result "{}", err I0421 03:07:22.935300 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:07:22.935329 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l I0421 03:07:22.936625 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l, ips: 10.128.0.7 I0421 03:07:22.936651 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:07:23.832195 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:07:23.832217 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:07:43.221224 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0421 03:08:10.490184 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0421 03:08:29.485876 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=71 I0421 03:08:31.804564 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0421 03:08:50.487662 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=136 I0421 03:09:03.597314 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0421 03:09:23.487808 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0421 03:09:29.386810 3808 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-cnbs7 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:09:29.386942 3808 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7 I0421 03:09:29.413229 3808 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7] pod update took 26.318694ms I0421 03:09:29.413252 3808 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7] addLogicalPort annotation time took 26.343172ms I0421 03:09:29.414453 3808 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7] addLogicalPort took 27.659082ms, libovsdb time 856.393µs I0421 03:09:29.744195 3808 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7 3b37a15d1db16dd9180d252e4c2593bf3ef0402a990890fcebf7d31f321c69a8 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7 3b37a15d1db16dd9180d252e4c2593bf3ef0402a990890fcebf7d31f321c69a8 network default NAD default NAD key ] I0421 03:09:29.754104 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-cnbs7, hostIfaceName: 3b37a15d1db16dd, network: default, NAD default, SandboxID: "3b37a15d1db16dd9180d252e4c2593bf3ef0402a990890fcebf7d31f321c69a8", PCI device ID: , UID: "c86ff667-341e-4561-bc8e-9299814915bd", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0421 03:09:29.803665 3808 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7 3b37a15d1db16dd9180d252e4c2593bf3ef0402a990890fcebf7d31f321c69a8 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cnbs7 3b37a15d1db16dd9180d252e4c2593bf3ef0402a990890fcebf7d31f321c69a8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:bf:af:dd:8f:bd\",\"name\":\"3b37a15d1db16dd\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8c1a616-fd89-455f-921d-d39697411de7\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:09:33.308463 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0421 03:09:35.488229 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=73 W0421 03:09:44.316036 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0421 03:09:44.348103 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0421 03:09:44.348185 3808 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-qmrz9 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-qmrz9: no pod IPs found I0421 03:09:46.024786 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0421 03:09:54.495840 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=610 I0421 03:10:00.174624 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29612350-mtpl7 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:10:00.174755 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 I0421 03:10:00.187530 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7] pod update took 12.800968ms I0421 03:10:00.187559 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7] addLogicalPort annotation time took 12.834993ms I0421 03:10:00.188804 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7] addLogicalPort took 14.198108ms, libovsdb time 937.045µs I0421 03:10:00.548174 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key ] I0421 03:10:00.557307 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29612350-mtpl7, hostIfaceName: 2c4fb995b3e1468, network: default, NAD default, SandboxID: "2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86", PCI device ID: , UID: "9b29d253-2356-4888-8927-93ea1fd11b30", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0421 03:10:00.608620 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:35:ae:de:e5:77\",\"name\":\"2c4fb995b3e1468\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e1ac9bc-fe5c-4b09-b607-816b5f7eeead\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:10:04.429563 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key ] I0421 03:10:04.506207 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 2c4fb995b3e1468f6b6f3caf80b963149a06823a47052b9ffaf63f75f3fc7d86 network default NAD default NAD key default], result "{}", err I0421 03:10:04.546681 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:10:04.546713 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 I0421 03:10:04.547925 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7, ips: 10.128.0.25 I0421 03:10:04.547957 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:10:05.440074 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:10:05.440126 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:10:11.699733 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0421 03:10:14.108655 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0421 03:10:17.911230 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0421 03:10:40.136048 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0421 03:13:31.494241 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=6 I0421 03:13:48.224842 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0421 03:13:56.266358 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0421 03:14:05.808483 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0421 03:14:34.491414 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=9 I0421 03:15:12.499454 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=15 I0421 03:15:28.620261 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0421 03:15:36.028273 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0421 03:15:40.111442 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0421 03:15:59.914385 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 I0421 03:16:02.246879 3808 namespace.go:98] [tsf] adding namespace I0421 03:16:02.247771 3808 namespace.go:102] [tsf] adding namespace took 870.573µs I0421 03:16:02.271873 3808 namespace.go:142] [tsf] updating namespace I0421 03:16:02.278096 3808 namespace.go:142] [tsf] updating namespace I0421 03:16:06.101060 3808 namespace.go:98] [konflux-ui] adding namespace I0421 03:16:06.101108 3808 namespace.go:98] [tsf-tpa] adding namespace I0421 03:16:06.102088 3808 namespace.go:102] [konflux-ui] adding namespace took 1.000112ms I0421 03:16:06.102850 3808 namespace.go:102] [tsf-tpa] adding namespace took 1.726312ms I0421 03:16:06.118191 3808 namespace.go:98] [rhtpa-operator] adding namespace I0421 03:16:06.118298 3808 namespace.go:98] [rhbk-operator] adding namespace I0421 03:16:06.118447 3808 namespace.go:98] [cert-manager-operator] adding namespace I0421 03:16:06.118618 3808 namespace.go:98] [tsf-tas] adding namespace I0421 03:16:06.118729 3808 namespace.go:98] [tsf-keycloak] adding namespace I0421 03:16:06.118946 3808 namespace.go:102] [rhtpa-operator] adding namespace took 740.029µs I0421 03:16:06.118974 3808 namespace.go:98] [konflux-operator] adding namespace I0421 03:16:06.119530 3808 namespace.go:102] [konflux-operator] adding namespace took 549.572µs I0421 03:16:06.120055 3808 namespace.go:102] [rhbk-operator] adding namespace took 1.738638ms I0421 03:16:06.120572 3808 namespace.go:102] [cert-manager-operator] adding namespace took 2.110026ms I0421 03:16:06.121070 3808 namespace.go:102] [tsf-tas] adding namespace took 2.426715ms I0421 03:16:06.121588 3808 namespace.go:102] [tsf-keycloak] adding namespace took 2.846079ms I0421 03:16:06.127950 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:16:06.169520 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:16:06.209632 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:16:06.221373 3808 namespace.go:142] [cert-manager-operator] updating namespace I0421 03:16:06.233523 3808 namespace.go:142] [rhbk-operator] updating namespace I0421 03:16:06.235260 3808 namespace.go:142] [rhtpa-operator] updating namespace I0421 03:16:06.293877 3808 namespace.go:142] [konflux-operator] updating namespace I0421 03:16:06.302236 3808 namespace.go:142] [tsf-tas] updating namespace I0421 03:16:06.303962 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:16:06.314724 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:16:06.372974 3808 namespace.go:142] [rhtpa-operator] updating namespace I0421 03:16:06.390015 3808 namespace.go:142] [rhbk-operator] updating namespace I0421 03:16:06.392323 3808 namespace.go:142] [rhtpa-operator] updating namespace I0421 03:16:06.404726 3808 namespace.go:142] [cert-manager-operator] updating namespace I0421 03:16:06.416407 3808 namespace.go:142] [rhbk-operator] updating namespace I0421 03:16:06.424867 3808 namespace.go:142] [konflux-operator] updating namespace I0421 03:16:06.431435 3808 namespace.go:142] [cert-manager-operator] updating namespace I0421 03:16:06.442425 3808 namespace.go:142] [tsf-tas] updating namespace I0421 03:16:06.442440 3808 namespace.go:142] [konflux-operator] updating namespace I0421 03:16:06.456485 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:16:06.462876 3808 namespace.go:142] [tsf-tas] updating namespace I0421 03:16:06.471439 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:16:06.477253 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:16:06.491735 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:16:07.139266 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:16:07.143207 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:16:12.600661 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0421 03:16:13.616029 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.616560 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.616586 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.616594 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0421 03:16:13.616609 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.616613 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0421 03:16:13.652598 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.653055 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.653077 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.653084 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0421 03:16:13.653094 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:13.653098 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0421 03:16:14.641675 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.641700 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.641714 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.641726 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ecer8vbm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.676771 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.676788 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.676800 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:14.676812 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2t86q8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:16.694133 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:16.694743 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:16.694771 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:16.694780 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0421 03:16:16.694794 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:16.694804 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0421 03:16:17.719500 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:17.719523 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:17.719542 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:17.719555 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac45682gwnw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:19.699298 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.699796 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.699820 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.699830 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0421 03:16:19.699841 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.699844 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0421 03:16:19.739354 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.739760 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.739781 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.739788 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0421 03:16:19.739798 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.739802 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0421 03:16:19.774253 3808 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.774647 3808 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.774672 3808 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.774680 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0421 03:16:19.774694 3808 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0421 03:16:19.774701 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0421 03:16:20.685886 3808 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-ndrzn for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:20.685997 3808 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn I0421 03:16:20.735812 3808 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn I0421 03:16:20.741039 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.741060 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.741076 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.741099 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7mzjq5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.759016 3808 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn] pod update took 73.048198ms I0421 03:16:20.759036 3808 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn] addLogicalPort annotation time took 73.072085ms I0421 03:16:20.760298 3808 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn] addLogicalPort took 74.428055ms, libovsdb time 859.751µs I0421 03:16:20.771978 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.771996 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.772008 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.772018 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.798792 3808 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.798811 3808 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.798823 3808 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:20.798832 3808 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dw9frx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:21.048275 3808 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn 6abeb0bd53ca5562039447572ad8a3ba0c1625affaa4738f7b6eebd9781188af network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn 6abeb0bd53ca5562039447572ad8a3ba0c1625affaa4738f7b6eebd9781188af network default NAD default NAD key ] I0421 03:16:21.057721 3808 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-ndrzn, hostIfaceName: 6abeb0bd53ca556, network: default, NAD default, SandboxID: "6abeb0bd53ca5562039447572ad8a3ba0c1625affaa4738f7b6eebd9781188af", PCI device ID: , UID: "5f6ccd8a-f1a4-47d2-a536-ec5b5a95ab1f", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0421 03:16:21.108006 3808 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn 6abeb0bd53ca5562039447572ad8a3ba0c1625affaa4738f7b6eebd9781188af network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-ndrzn 6abeb0bd53ca5562039447572ad8a3ba0c1625affaa4738f7b6eebd9781188af network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:1f:ee:2e:c2:c6\",\"name\":\"6abeb0bd53ca556\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d50bec61-e498-4fb7-b4fc-a6c63a87783a\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:16:23.270718 3808 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm] creating logical port tsf-tpa_rhtpa-operator-controller-manager-576c85d8f-8bwwm for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:23.270881 3808 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm I0421 03:16:23.285837 3808 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm] pod update took 14.983588ms I0421 03:16:23.285866 3808 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm] addLogicalPort annotation time took 15.017144ms I0421 03:16:23.287367 3808 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm] addLogicalPort took 16.705713ms, libovsdb time 1.057098ms I0421 03:16:23.639186 3808 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm d90f5723fe0753709526baf5cbf36fa2dbfb965e10966ffa4f8bf0df90bf32a2 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm d90f5723fe0753709526baf5cbf36fa2dbfb965e10966ffa4f8bf0df90bf32a2 network default NAD default NAD key ] I0421 03:16:23.650658 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-576c85d8f-8bwwm, hostIfaceName: d90f5723fe07537, network: default, NAD default, SandboxID: "d90f5723fe0753709526baf5cbf36fa2dbfb965e10966ffa4f8bf0df90bf32a2", PCI device ID: , UID: "6d751d42-6a87-41d2-b4cc-8b6b2cc9850e", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0421 03:16:23.702007 3808 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm d90f5723fe0753709526baf5cbf36fa2dbfb965e10966ffa4f8bf0df90bf32a2 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-576c85d8f-8bwwm d90f5723fe0753709526baf5cbf36fa2dbfb965e10966ffa4f8bf0df90bf32a2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:c4:69:5b:c3:fe\",\"name\":\"d90f5723fe07537\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85cbb4c5-6219-4b98-8b06-155cd5203efc\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:16:27.895882 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0421 03:16:31.577927 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0421 03:16:31.591237 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0421 03:16:35.489700 3808 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv] creating logical port konflux-operator_konflux-operator-controller-manager-586bdd7c4b-nmjgv for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:35.489809 3808 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv I0421 03:16:35.521965 3808 pod.go:62] [konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv] pod update took 32.18122ms I0421 03:16:35.522004 3808 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv] addLogicalPort annotation time took 32.206644ms I0421 03:16:35.523343 3808 pods.go:271] [konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv] addLogicalPort took 33.659124ms, libovsdb time 973.788µs I0421 03:16:35.848672 3808 cni.go:452] [konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv 8ea0146cf800144cd3ecb6fea324da11c4e21df0c22702b8fdffeba6a5c05648 network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv 8ea0146cf800144cd3ecb6fea324da11c4e21df0c22702b8fdffeba6a5c05648 network default NAD default NAD key ] I0421 03:16:35.858430 3808 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-586bdd7c4b-nmjgv, hostIfaceName: 8ea0146cf800144, network: default, NAD default, SandboxID: "8ea0146cf800144cd3ecb6fea324da11c4e21df0c22702b8fdffeba6a5c05648", PCI device ID: , UID: "4307d1d4-5d15-418f-beef-3bd7dd856aca", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0421 03:16:35.907543 3808 cni.go:473] [konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv 8ea0146cf800144cd3ecb6fea324da11c4e21df0c22702b8fdffeba6a5c05648 network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-586bdd7c4b-nmjgv 8ea0146cf800144cd3ecb6fea324da11c4e21df0c22702b8fdffeba6a5c05648 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:51:36:e0:e0:b9\",\"name\":\"8ea0146cf800144\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d478eb7c-630d-4f65-a372-84af8f61fa8f\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:16:36.436283 3808 namespace.go:142] [openshift-operators] updating namespace I0421 03:16:36.466245 3808 namespace.go:142] [openshift-operators] updating namespace I0421 03:16:37.280971 3808 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-pnjk9 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:37.281108 3808 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9 I0421 03:16:37.304183 3808 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9] pod update took 23.108215ms I0421 03:16:37.304202 3808 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9] addLogicalPort annotation time took 23.131289ms I0421 03:16:37.305397 3808 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9] addLogicalPort took 24.448191ms, libovsdb time 889.216µs I0421 03:16:37.491373 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=13 I0421 03:16:38.132382 3808 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9 a77655ae49cb3e2f86ba039ecd4306166fd7fa812c555ca4bf9bf8785c68f89b network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9 a77655ae49cb3e2f86ba039ecd4306166fd7fa812c555ca4bf9bf8785c68f89b network default NAD default NAD key ] I0421 03:16:38.144263 3808 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-pnjk9, hostIfaceName: a77655ae49cb3e2, network: default, NAD default, SandboxID: "a77655ae49cb3e2f86ba039ecd4306166fd7fa812c555ca4bf9bf8785c68f89b", PCI device ID: , UID: "a9afb0ce-37be-4974-bb45-6c3eded66d18", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0421 03:16:38.194892 3808 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9 a77655ae49cb3e2f86ba039ecd4306166fd7fa812c555ca4bf9bf8785c68f89b network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pnjk9 a77655ae49cb3e2f86ba039ecd4306166fd7fa812c555ca4bf9bf8785c68f89b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:21:fe:c2:f6:c1\",\"name\":\"a77655ae49cb3e2\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fba60cdd-c52a-4ddd-a95b-fa86be0b0b95\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:16:38.916032 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0421 03:16:38.970799 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0421 03:16:40.193394 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0421 03:16:40.198036 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0421 03:16:40.229404 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0421 03:16:40.231076 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0421 03:16:41.997199 3808 namespace.go:98] [cert-manager] adding namespace I0421 03:16:41.997992 3808 namespace.go:102] [cert-manager] adding namespace took 768.226µs I0421 03:16:42.031467 3808 namespace.go:142] [cert-manager] updating namespace I0421 03:16:42.040231 3808 namespace.go:142] [cert-manager] updating namespace I0421 03:16:43.991017 3808 namespace.go:142] [openshift-operators] updating namespace I0421 03:16:44.375397 3808 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-rf7gv for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:44.375532 3808 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv I0421 03:16:44.401356 3808 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv] pod update took 25.856556ms I0421 03:16:44.401386 3808 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv] addLogicalPort annotation time took 25.89147ms I0421 03:16:44.402649 3808 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv] addLogicalPort took 27.280184ms, libovsdb time 857.801µs I0421 03:16:44.743961 3808 cni.go:452] [openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv ccabe5f8a7307a43223128bda4d24082d1d37edacafe16eccbec31bc7cebe345 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv ccabe5f8a7307a43223128bda4d24082d1d37edacafe16eccbec31bc7cebe345 network default NAD default NAD key ] I0421 03:16:44.755030 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-rf7gv, hostIfaceName: ccabe5f8a7307a4, network: default, NAD default, SandboxID: "ccabe5f8a7307a43223128bda4d24082d1d37edacafe16eccbec31bc7cebe345", PCI device ID: , UID: "2161ccb4-e54e-433d-9e3b-0e903acfedb8", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0421 03:16:44.810850 3808 cni.go:473] [openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv ccabe5f8a7307a43223128bda4d24082d1d37edacafe16eccbec31bc7cebe345 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-rf7gv ccabe5f8a7307a43223128bda4d24082d1d37edacafe16eccbec31bc7cebe345 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:c4:3a:09:50:91\",\"name\":\"ccabe5f8a7307a4\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9311f03e-94d2-4436-bcf1-53bc227209a7\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:16:48.715253 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0421 03:16:48.715273 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook I0421 03:16:49.626992 3808 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-cggnc for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:49.627124 3808 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc I0421 03:16:49.708237 3808 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc] pod update took 81.137698ms I0421 03:16:49.708266 3808 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc] addLogicalPort annotation time took 81.175818ms I0421 03:16:49.709790 3808 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc] addLogicalPort took 82.815823ms, libovsdb time 1.088376ms W0421 03:16:50.087425 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0421 03:16:50.279534 3808 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc 7882bc3c9422ca86c149de2fad3c2f0d128345fc09813866025ff409f4c6ed09 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc 7882bc3c9422ca86c149de2fad3c2f0d128345fc09813866025ff409f4c6ed09 network default NAD default NAD key ] I0421 03:16:50.290443 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-cggnc, hostIfaceName: 7882bc3c9422ca8, network: default, NAD default, SandboxID: "7882bc3c9422ca86c149de2fad3c2f0d128345fc09813866025ff409f4c6ed09", PCI device ID: , UID: "efa58a7d-441b-40d7-9169-a31deec8e014", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0421 03:16:50.301457 3808 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-fdsm7 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:16:50.301564 3808 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7 I0421 03:16:50.348834 3808 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc 7882bc3c9422ca86c149de2fad3c2f0d128345fc09813866025ff409f4c6ed09 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cggnc 7882bc3c9422ca86c149de2fad3c2f0d128345fc09813866025ff409f4c6ed09 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:d4:01:99:44:83\",\"name\":\"7882bc3c9422ca8\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ce6069bc-9c19-44d8-9441-31558a75e045\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:16:50.352404 3808 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7] pod update took 50.874233ms I0421 03:16:50.352434 3808 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7] addLogicalPort annotation time took 50.910131ms I0421 03:16:50.353775 3808 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7] addLogicalPort took 52.333373ms, libovsdb time 918.104µs I0421 03:16:50.666711 3808 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7 fee218dd77d7af98576a1749d47c3df74af38e2bafc23e2695130e30b68c77b1 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7 fee218dd77d7af98576a1749d47c3df74af38e2bafc23e2695130e30b68c77b1 network default NAD default NAD key ] I0421 03:16:50.678591 3808 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-fdsm7, hostIfaceName: fee218dd77d7af9, network: default, NAD default, SandboxID: "fee218dd77d7af98576a1749d47c3df74af38e2bafc23e2695130e30b68c77b1", PCI device ID: , UID: "541e6799-1ff5-4498-b37e-2460cc6ccf40", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0421 03:16:50.738374 3808 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7 fee218dd77d7af98576a1749d47c3df74af38e2bafc23e2695130e30b68c77b1 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-fdsm7 fee218dd77d7af98576a1749d47c3df74af38e2bafc23e2695130e30b68c77b1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:84:b5:7c:ba:0a\",\"name\":\"fee218dd77d7af9\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26d56caf-30e0-44d3-ac24-35ceb0bc90de\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:16:52.159672 3808 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:52.160338 3808 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:16:52.193566 3808 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:52.193589 3808 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:16:52.209717 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0421 03:16:52.209756 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0421 03:16:53.815189 3808 namespace.go:98] [trusted-artifact-signer] adding namespace I0421 03:16:53.816277 3808 namespace.go:102] [trusted-artifact-signer] adding namespace took 1.0604ms I0421 03:16:53.869791 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0421 03:16:53.908058 3808 namespace.go:142] [trusted-artifact-signer] updating namespace W0421 03:16:54.094406 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0421 03:16:59.564367 3808 namespace.go:98] [openshift-pipelines] adding namespace I0421 03:16:59.565287 3808 namespace.go:102] [openshift-pipelines] adding namespace took 898.629µs I0421 03:16:59.583613 3808 namespace.go:142] [openshift-pipelines] updating namespace I0421 03:17:00.297057 3808 namespace.go:142] [cert-manager] updating namespace I0421 03:17:00.490298 3808 namespace.go:142] [cert-manager-operator] updating namespace I0421 03:17:00.892126 3808 namespace.go:142] [konflux-operator] updating namespace I0421 03:17:01.090039 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:17:01.292357 3808 namespace.go:142] [open-cluster-management-2pq9drt9viog8225llnhb4eduucokg20] updating namespace I0421 03:17:01.692574 3808 namespace.go:142] [rhbk-operator] updating namespace I0421 03:17:01.893287 3808 namespace.go:142] [rhtpa-operator] updating namespace I0421 03:17:02.096573 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0421 03:17:02.302000 3808 namespace.go:142] [tsf] updating namespace I0421 03:17:02.497732 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:17:02.704493 3808 namespace.go:142] [tsf-tas] updating namespace I0421 03:17:02.722313 3808 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-99nnb] creating logical port cert-manager_cert-manager-59c77684cc-99nnb for pod on switch ip-10-0-2-88.ec2.internal I0421 03:17:02.722441 3808 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-99nnb I0421 03:17:02.737282 3808 pod.go:62] [cert-manager/cert-manager-59c77684cc-99nnb] pod update took 14.868443ms I0421 03:17:02.737306 3808 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-99nnb] addLogicalPort annotation time took 14.918923ms I0421 03:17:02.738554 3808 pods.go:271] [cert-manager/cert-manager-59c77684cc-99nnb] addLogicalPort took 16.265299ms, libovsdb time 863.626µs I0421 03:17:02.901280 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:17:03.027952 3808 namespace.go:142] [cert-manager] updating namespace I0421 03:17:03.036429 3808 namespace.go:142] [cert-manager-operator] updating namespace I0421 03:17:03.046869 3808 namespace.go:142] [default] updating namespace I0421 03:17:03.055126 3808 namespace.go:142] [konflux-operator] updating namespace I0421 03:17:03.064718 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:17:03.073537 3808 namespace.go:142] [open-cluster-management-2pq9drt9viog8225llnhb4eduucokg20] updating namespace I0421 03:17:03.084456 3808 namespace.go:142] [openshift] updating namespace I0421 03:17:03.092466 3808 namespace.go:142] [rhbk-operator] updating namespace I0421 03:17:03.092851 3808 cni.go:452] [cert-manager/cert-manager-59c77684cc-99nnb 58dd01a8851fa453879f0095be2d58525beef4b0f2e29b604e1bde5b629b40e8 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-99nnb 58dd01a8851fa453879f0095be2d58525beef4b0f2e29b604e1bde5b629b40e8 network default NAD default NAD key ] I0421 03:17:03.102918 3808 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-99nnb, hostIfaceName: 58dd01a8851fa45, network: default, NAD default, SandboxID: "58dd01a8851fa453879f0095be2d58525beef4b0f2e29b604e1bde5b629b40e8", PCI device ID: , UID: "1dee4cce-e635-4d02-8895-335bcf5e4b44", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0421 03:17:03.103179 3808 namespace.go:142] [rhtpa-operator] updating namespace I0421 03:17:03.110993 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0421 03:17:03.121594 3808 namespace.go:142] [tsf] updating namespace I0421 03:17:03.142240 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:17:03.159062 3808 namespace.go:142] [tsf-tas] updating namespace I0421 03:17:03.162840 3808 cni.go:473] [cert-manager/cert-manager-59c77684cc-99nnb 58dd01a8851fa453879f0095be2d58525beef4b0f2e29b604e1bde5b629b40e8 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-99nnb 58dd01a8851fa453879f0095be2d58525beef4b0f2e29b604e1bde5b629b40e8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:05:c2:df:ae:3b\",\"name\":\"58dd01a8851fa45\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c34e5bb-d6ee-410a-af5d-432ac8db922a\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:17:03.179796 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:17:03.280024 3808 namespace.go:142] [cert-manager] updating namespace I0421 03:17:03.382764 3808 namespace.go:142] [cert-manager-operator] updating namespace I0421 03:17:03.486824 3808 namespace.go:142] [default] updating namespace I0421 03:17:03.580376 3808 namespace.go:142] [konflux-operator] updating namespace I0421 03:17:03.683301 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:17:03.782457 3808 namespace.go:142] [open-cluster-management-2pq9drt9viog8225llnhb4eduucokg20] updating namespace I0421 03:17:03.883590 3808 namespace.go:142] [openshift] updating namespace I0421 03:17:03.991091 3808 namespace.go:142] [rhbk-operator] updating namespace I0421 03:17:04.081184 3808 namespace.go:142] [rhtpa-operator] updating namespace I0421 03:17:04.179416 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0421 03:17:04.281890 3808 namespace.go:142] [tsf] updating namespace I0421 03:17:04.385115 3808 namespace.go:142] [tsf-keycloak] updating namespace I0421 03:17:04.481673 3808 namespace.go:142] [tsf-tas] updating namespace I0421 03:17:04.582345 3808 namespace.go:142] [tsf-tpa] updating namespace I0421 03:17:04.858182 3808 namespace.go:142] [openshift-pipelines] updating namespace W0421 03:17:05.072626 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0421 03:17:06.220102 3808 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0421 03:17:06.391854 3808 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0421 03:17:15.475926 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p for pod on switch ip-10-0-2-88.ec2.internal I0421 03:17:15.476038 3808 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p I0421 03:17:15.494345 3808 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p] pod update took 18.331313ms I0421 03:17:15.494375 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p] addLogicalPort annotation time took 18.355812ms I0421 03:17:15.495436 3808 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p] addLogicalPort took 19.52747ms, libovsdb time 758.571µs I0421 03:17:15.576555 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-flfps for pod on switch ip-10-0-2-88.ec2.internal I0421 03:17:15.576637 3808 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps I0421 03:17:15.593774 3808 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps] pod update took 17.162437ms I0421 03:17:15.593795 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps] addLogicalPort annotation time took 17.187709ms I0421 03:17:15.594991 3808 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps] addLogicalPort took 18.449195ms, libovsdb time 877.763µs I0421 03:17:15.844016 3808 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p d66e73e4c3292446f60896d52eb7a5bc5ed2849c3fb9fe9786c302a8ab0128ce network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p d66e73e4c3292446f60896d52eb7a5bc5ed2849c3fb9fe9786c302a8ab0128ce network default NAD default NAD key ] I0421 03:17:15.854470 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p, hostIfaceName: d66e73e4c329244, network: default, NAD default, SandboxID: "d66e73e4c3292446f60896d52eb7a5bc5ed2849c3fb9fe9786c302a8ab0128ce", PCI device ID: , UID: "e26c9cfc-9807-490a-a1cf-2ab53f48efee", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0421 03:17:15.907302 3808 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p d66e73e4c3292446f60896d52eb7a5bc5ed2849c3fb9fe9786c302a8ab0128ce network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-h2t2p d66e73e4c3292446f60896d52eb7a5bc5ed2849c3fb9fe9786c302a8ab0128ce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:08:d6:ab:e1:67\",\"name\":\"d66e73e4c329244\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/900bf706-0405-4cee-8733-56b56668c8f6\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:17:15.939216 3808 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps 4fcf75a855cce62baa43e6d7b28c75dc4d4a7aef38fd38790a9126f1f526681d network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps 4fcf75a855cce62baa43e6d7b28c75dc4d4a7aef38fd38790a9126f1f526681d network default NAD default NAD key ] I0421 03:17:15.948780 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-flfps, hostIfaceName: 4fcf75a855cce62, network: default, NAD default, SandboxID: "4fcf75a855cce62baa43e6d7b28c75dc4d4a7aef38fd38790a9126f1f526681d", PCI device ID: , UID: "77f26fff-d690-4781-a634-b262fa6d6266", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0421 03:17:16.000248 3808 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps 4fcf75a855cce62baa43e6d7b28c75dc4d4a7aef38fd38790a9126f1f526681d network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-flfps 4fcf75a855cce62baa43e6d7b28c75dc4d4a7aef38fd38790a9126f1f526681d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:d2:a0:2b:32:5c\",\"name\":\"4fcf75a855cce62\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c74f7454-47f9-4ecb-8d9b-1190667ac7a2\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:17:19.914248 3808 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0421 03:17:19.914267 3808 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0421 03:17:19.914272 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0421 03:17:19.939332 3808 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0421 03:17:19.939345 3808 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0421 03:17:19.939351 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0421 03:17:21.177611 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:21.177636 3808 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:21.177643 3808 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:21.177650 3808 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0421 03:17:31.139229 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 W0421 03:17:32.190236 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:32.190259 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:32.190267 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:32.190272 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0421 03:17:43.257442 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0421 03:17:45.351064 3808 namespace.go:142] [openshift-pipelines] updating namespace I0421 03:17:49.418033 3808 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:17:49.418604 3808 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:17:49.439423 3808 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:17:49.439445 3808 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:17:49.442724 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0421 03:17:49.442751 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0421 03:17:51.409858 3808 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0421 03:17:51.410568 3808 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0421 03:17:51.410595 3808 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0421 03:17:51.412616 3808 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0421 03:17:51.756281 3808 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:17:51.756434 3808 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0421 03:17:51.773121 3808 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 16.727767ms I0421 03:17:51.773155 3808 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 16.754771ms I0421 03:17:51.774329 3808 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 18.069719ms, libovsdb time 842.27µs I0421 03:17:52.216673 3808 cni.go:452] [tsf-keycloak/keycloak-0 51b105eddcfec2fe36ff28ab57ac5bbec95e683d59381a6500f748654869696c network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 51b105eddcfec2fe36ff28ab57ac5bbec95e683d59381a6500f748654869696c network default NAD default NAD key ] I0421 03:17:52.226861 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 51b105eddcfec2f, network: default, NAD default, SandboxID: "51b105eddcfec2fe36ff28ab57ac5bbec95e683d59381a6500f748654869696c", PCI device ID: , UID: "2a45fdf9-e2a2-47e2-abf3-fb715cf01cab", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0421 03:17:52.279029 3808 cni.go:473] [tsf-keycloak/keycloak-0 51b105eddcfec2fe36ff28ab57ac5bbec95e683d59381a6500f748654869696c network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 51b105eddcfec2fe36ff28ab57ac5bbec95e683d59381a6500f748654869696c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:a5:b8:a8:67:17\",\"name\":\"51b105eddcfec2f\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2559e574-a052-487c-ad6c-83228fcef43e\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:17:56.490891 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0421 03:18:01.209397 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0421 03:18:02.381728 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0421 03:18:02.381751 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0421 03:18:02.382435 3808 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0421 03:18:02.382451 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0421 03:18:15.490096 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=57 W0421 03:18:25.475131 3808 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0421 03:18:25.475165 3808 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0421 03:18:25.475170 3808 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0421 03:18:28.432771 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0421 03:18:28.432836 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0421 03:18:28.432968 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0421 03:18:28.432983 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service I0421 03:18:28.704065 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0421 03:18:31.650082 3808 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-bbzcg] creating logical port tsf-keycloak_tsf-iam-bbzcg for pod on switch ip-10-0-2-88.ec2.internal I0421 03:18:31.650198 3808 kube.go:256] Updating pod tsf-keycloak/tsf-iam-bbzcg I0421 03:18:31.664370 3808 pod.go:62] [tsf-keycloak/tsf-iam-bbzcg] pod update took 14.199354ms I0421 03:18:31.664395 3808 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-bbzcg] addLogicalPort annotation time took 14.230866ms I0421 03:18:31.665760 3808 pods.go:271] [tsf-keycloak/tsf-iam-bbzcg] addLogicalPort took 15.699728ms, libovsdb time 959.983µs I0421 03:18:32.016319 3808 cni.go:452] [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key ] I0421 03:18:32.026675 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-bbzcg, hostIfaceName: 9ae66f191c2b84e, network: default, NAD default, SandboxID: "9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc", PCI device ID: , UID: "a2c018fa-0d5f-47bf-830d-efc7a26d17ea", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0421 03:18:32.087669 3808 cni.go:473] [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:b0:cc:b7:c2:ca\",\"name\":\"9ae66f191c2b84e\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f397cf4a-3af2-4c45-9394-fdf3f2a666cf\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:18:44.712758 3808 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:18:44.713446 3808 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:18:44.753820 3808 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:18:44.753853 3808 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:18:44.757065 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0421 03:18:44.757093 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0421 03:18:51.627181 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0421 03:18:51.627202 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0421 03:18:51.650472 3808 cni.go:452] [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key ] I0421 03:18:51.750195 3808 cni.go:473] [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-bbzcg 9ae66f191c2b84ef7987585beb34fc82574694c5681868f886fd9b00e8e6c4dc network default NAD default NAD key default], result "{}", err I0421 03:18:51.802112 3808 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-bbzcg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:18:51.802167 3808 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-bbzcg I0421 03:18:51.803630 3808 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-bbzcg, ips: 10.128.0.37 I0421 03:18:51.803664 3808 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-bbzcg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:18:52.672649 3808 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-bbzcg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:18:52.672680 3808 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-bbzcg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0421 03:18:56.411924 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0421 03:18:56.436685 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0421 03:19:03.844801 3808 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-tkgl7-69l5d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:03.845470 3808 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-tkgl7-69l5d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:04.267664 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-l2c2f for pod on switch ip-10-0-2-88.ec2.internal I0421 03:19:04.267771 3808 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f I0421 03:19:04.286667 3808 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f] pod update took 18.924018ms I0421 03:19:04.286689 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f] addLogicalPort annotation time took 18.950183ms I0421 03:19:04.287907 3808 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f] addLogicalPort took 20.262782ms, libovsdb time 850.179µs I0421 03:19:04.364645 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-rpmc9 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:19:04.364777 3808 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9 I0421 03:19:04.379290 3808 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9] pod update took 14.537758ms I0421 03:19:04.379314 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9] addLogicalPort annotation time took 14.56609ms I0421 03:19:04.380597 3808 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9] addLogicalPort took 15.972077ms, libovsdb time 854.102µs I0421 03:19:04.465810 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-76jhm for pod on switch ip-10-0-2-88.ec2.internal I0421 03:19:04.465917 3808 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm I0421 03:19:04.481274 3808 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm] pod update took 15.379299ms I0421 03:19:04.481295 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm] addLogicalPort annotation time took 15.403617ms I0421 03:19:04.482515 3808 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm] addLogicalPort took 16.724851ms, libovsdb time 854.918µs I0421 03:19:04.628471 3808 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f 363ca368f29f681ab1450b7f626655f4fcff217c3205eba99a3ad7c4c230e07d network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f 363ca368f29f681ab1450b7f626655f4fcff217c3205eba99a3ad7c4c230e07d network default NAD default NAD key ] I0421 03:19:04.638040 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-l2c2f, hostIfaceName: 363ca368f29f681, network: default, NAD default, SandboxID: "363ca368f29f681ab1450b7f626655f4fcff217c3205eba99a3ad7c4c230e07d", PCI device ID: , UID: "449d54ba-f187-458c-84d9-0287d2a1cd66", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0421 03:19:04.692906 3808 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f 363ca368f29f681ab1450b7f626655f4fcff217c3205eba99a3ad7c4c230e07d network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-l2c2f 363ca368f29f681ab1450b7f626655f4fcff217c3205eba99a3ad7c4c230e07d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:e7:b0:72:eb:4d\",\"name\":\"363ca368f29f681\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/09c0a219-88c0-40ef-92c5-9a118130f359\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:19:04.726576 3808 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9 6c74717dba743afd7e4c39462bf649f0d04e315386598d90ca532d6b0ed086a8 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9 6c74717dba743afd7e4c39462bf649f0d04e315386598d90ca532d6b0ed086a8 network default NAD default NAD key ] I0421 03:19:04.736389 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-rpmc9, hostIfaceName: 6c74717dba743af, network: default, NAD default, SandboxID: "6c74717dba743afd7e4c39462bf649f0d04e315386598d90ca532d6b0ed086a8", PCI device ID: , UID: "8da8a64f-2274-4ccc-8de5-051769684533", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0421 03:19:04.788920 3808 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9 6c74717dba743afd7e4c39462bf649f0d04e315386598d90ca532d6b0ed086a8 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-rpmc9 6c74717dba743afd7e4c39462bf649f0d04e315386598d90ca532d6b0ed086a8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:79:55:85:5b:37\",\"name\":\"6c74717dba743af\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e11d5b6-dc60-4bb2-bfbc-0887d3a7ee97\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:19:04.825464 3808 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm 40c584db1073b05d8fdf57d8b8e1b85ddad5ce3768139aeece07e8097a4baeaf network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm 40c584db1073b05d8fdf57d8b8e1b85ddad5ce3768139aeece07e8097a4baeaf network default NAD default NAD key ] I0421 03:19:04.835716 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-76jhm, hostIfaceName: 40c584db1073b05, network: default, NAD default, SandboxID: "40c584db1073b05d8fdf57d8b8e1b85ddad5ce3768139aeece07e8097a4baeaf", PCI device ID: , UID: "b7246012-7ec1-40e8-ac43-082c8b9c3380", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0421 03:19:04.868268 3808 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-tkgl7-69l5d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:04.868294 3808 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-tkgl7-69l5d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:04.889535 3808 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm 40c584db1073b05d8fdf57d8b8e1b85ddad5ce3768139aeece07e8097a4baeaf network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-76jhm 40c584db1073b05d8fdf57d8b8e1b85ddad5ce3768139aeece07e8097a4baeaf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:7d:c5:4c:b8:4f\",\"name\":\"40c584db1073b05\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9a1d913-83b6-454b-b51e-10b6ccf6b4ee\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:19:07.759315 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0421 03:19:07.784664 3808 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0421 03:19:07.784683 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0421 03:19:08.500997 3808 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0421 03:19:08.751756 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0421 03:19:08.751780 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0421 03:19:18.747985 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0421 03:19:19.823953 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-ccqfc for pod on switch ip-10-0-2-88.ec2.internal I0421 03:19:19.824047 3808 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc I0421 03:19:19.839880 3808 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc] pod update took 15.85793ms I0421 03:19:19.839901 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc] addLogicalPort annotation time took 15.883538ms I0421 03:19:19.841251 3808 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc] addLogicalPort took 17.315755ms, libovsdb time 911.326µs I0421 03:19:20.783031 3808 cni.go:452] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc 713643db762dc19acaf86b7681d54fba0174c5b2c2d3bdf19db5bcb1bd060102 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc 713643db762dc19acaf86b7681d54fba0174c5b2c2d3bdf19db5bcb1bd060102 network default NAD default NAD key ] I0421 03:19:20.792434 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-ccqfc, hostIfaceName: 713643db762dc19, network: default, NAD default, SandboxID: "713643db762dc19acaf86b7681d54fba0174c5b2c2d3bdf19db5bcb1bd060102", PCI device ID: , UID: "16349af5-b0e9-4e64-9418-cf5d23963287", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0421 03:19:20.844231 3808 cni.go:473] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc 713643db762dc19acaf86b7681d54fba0174c5b2c2d3bdf19db5bcb1bd060102 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-ccqfc 713643db762dc19acaf86b7681d54fba0174c5b2c2d3bdf19db5bcb1bd060102 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:b3:2f:47:54:9b\",\"name\":\"713643db762dc19\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5b364337-a19a-4aa3-9167-da5244b0471a\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:19:37.028346 3808 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:37.028955 3808 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:37.048746 3808 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:37.048764 3808 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:37.055103 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0421 03:19:37.055123 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0421 03:19:45.045881 3808 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-nzsld-8br7n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:45.046426 3808 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-nzsld-8br7n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:45.926064 3808 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-nzsld-8br7n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:45.926084 3808 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-nzsld-8br7n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:46.070702 3808 obj_retry.go:498] Detected object tsf/patch-tekton-config-hwjw8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:46.071269 3808 obj_retry.go:498] Detected object tsf/patch-tekton-config-hwjw8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:47.092660 3808 obj_retry.go:492] Detected object tsf/patch-tekton-config-hwjw8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:47.092682 3808 obj_retry.go:492] Detected object tsf/patch-tekton-config-hwjw8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:48.059178 3808 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-ckzqn-v69rm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:48.059763 3808 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-ckzqn-v69rm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:48.944860 3808 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-ckzqn-v69rm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:48.944912 3808 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-ckzqn-v69rm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:49.044563 3808 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w] creating logical port tsf-tas_rekor-search-ui-6b9fcc876c-2hf7w for pod on switch ip-10-0-2-88.ec2.internal I0421 03:19:49.044643 3808 kube.go:256] Updating pod tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w I0421 03:19:49.062034 3808 pod.go:62] [tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w] pod update took 17.409522ms I0421 03:19:49.062054 3808 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w] addLogicalPort annotation time took 17.436167ms I0421 03:19:49.063299 3808 pods.go:271] [tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w] addLogicalPort took 18.75079ms, libovsdb time 833.941µs I0421 03:19:49.406709 3808 cni.go:452] [tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w 2e731db04f186fe40c1ddc10874692ff56ca4359cc2a966e08e7484e2745906e network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w 2e731db04f186fe40c1ddc10874692ff56ca4359cc2a966e08e7484e2745906e network default NAD default NAD key ] I0421 03:19:49.416242 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-6b9fcc876c-2hf7w, hostIfaceName: 2e731db04f186fe, network: default, NAD default, SandboxID: "2e731db04f186fe40c1ddc10874692ff56ca4359cc2a966e08e7484e2745906e", PCI device ID: , UID: "1ea57197-ee32-498d-9488-8195610a4786", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0421 03:19:49.470638 3808 cni.go:473] [tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w 2e731db04f186fe40c1ddc10874692ff56ca4359cc2a966e08e7484e2745906e network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-6b9fcc876c-2hf7w 2e731db04f186fe40c1ddc10874692ff56ca4359cc2a966e08e7484e2745906e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:6c:0a:8d:4b:5c\",\"name\":\"2e731db04f186fe\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae125eff-1318-4d80-afb8-b37cfee49682\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:19:51.955320 3808 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0421 03:19:51.982553 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0421 03:19:53.093463 3808 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-wfn4f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:53.093989 3808 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-wfn4f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:53.981637 3808 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-wfn4f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:53.981660 3808 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-wfn4f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:54.016613 3808 obj_retry.go:492] Detected object tsf/patch-tekton-config-hwjw8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:54.016636 3808 obj_retry.go:492] Detected object tsf/patch-tekton-config-hwjw8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:54.019202 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-hwjw8 I0421 03:19:54.019230 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-hwjw8 W0421 03:19:54.754619 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0421 03:19:58.123573 3808 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:58.124212 3808 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:19:58.144409 3808 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:58.144426 3808 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:19:58.151741 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0421 03:19:58.151762 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0421 03:20:00.021402 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0421 03:20:00.166273 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29612360-6pbj9 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:20:00.166362 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 I0421 03:20:00.178963 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9] pod update took 12.626359ms I0421 03:20:00.178984 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9] addLogicalPort annotation time took 12.651493ms I0421 03:20:00.180229 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9] addLogicalPort took 13.976594ms, libovsdb time 824.159µs I0421 03:20:00.695135 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key ] I0421 03:20:00.704857 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29612360-6pbj9, hostIfaceName: 2d965038b239a30, network: default, NAD default, SandboxID: "2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c", PCI device ID: , UID: "3447e1c1-05ad-4a31-8c33-3819c330066c", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0421 03:20:00.756187 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:62:a6:8e:99:d6\",\"name\":\"2d965038b239a30\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2a9f232f-f338-47dc-8543-5162d6dca9c3\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:20:03.994103 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0421 03:20:04.051393 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key ] I0421 03:20:04.133400 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 2d965038b239a302fd2bbce49a7754cddcb0a7fe31cc1feef2295fb720f8516c network default NAD default NAD key default], result "{}", err I0421 03:20:04.180215 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:04.180241 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 I0421 03:20:04.181570 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9, ips: 10.128.0.43 I0421 03:20:04.181596 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:04.188345 3808 obj_retry.go:498] Detected object tsf-tpa/create-db-f6pgz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:04.188709 3808 obj_retry.go:498] Detected object tsf-tpa/create-db-f6pgz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:05.058131 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:05.058174 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612360-6pbj9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:05.212749 3808 obj_retry.go:492] Detected object tsf-tpa/create-db-f6pgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:05.212775 3808 obj_retry.go:492] Detected object tsf-tpa/create-db-f6pgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:08.212833 3808 obj_retry.go:498] Detected object tsf-tpa/migrate-db-bfpfm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:08.213421 3808 obj_retry.go:498] Detected object tsf-tpa/migrate-db-bfpfm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:08.352032 3808 obj_retry.go:492] Detected object tsf-tpa/migrate-db-bfpfm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:08.352050 3808 obj_retry.go:492] Detected object tsf-tpa/migrate-db-bfpfm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:14.004547 3808 obj_retry.go:498] Detected object tsf-tpa/create-importers-dhltm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:14.005097 3808 obj_retry.go:498] Detected object tsf-tpa/create-importers-dhltm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:15.028975 3808 obj_retry.go:492] Detected object tsf-tpa/create-importers-dhltm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:15.028998 3808 obj_retry.go:492] Detected object tsf-tpa/create-importers-dhltm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:15.295967 3808 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-9j75t-rdbk4] creating logical port tsf-tas_tuf-repository-init-9j75t-rdbk4 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:20:15.296077 3808 kube.go:256] Updating pod tsf-tas/tuf-repository-init-9j75t-rdbk4 I0421 03:20:15.309037 3808 pod.go:62] [tsf-tas/tuf-repository-init-9j75t-rdbk4] pod update took 12.993994ms I0421 03:20:15.309059 3808 base_network_controller_pods.go:951] [default/tsf-tas/tuf-repository-init-9j75t-rdbk4] addLogicalPort annotation time took 13.021706ms I0421 03:20:15.310403 3808 pods.go:271] [tsf-tas/tuf-repository-init-9j75t-rdbk4] addLogicalPort took 14.452927ms, libovsdb time 925.44µs I0421 03:20:19.255408 3808 cni.go:452] [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key ] I0421 03:20:19.265285 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-9j75t-rdbk4, hostIfaceName: 69a37657c825003, network: default, NAD default, SandboxID: "69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3", PCI device ID: , UID: "d21b8dee-31cf-4a42-a013-adaf796d0bfa", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0421 03:20:19.319760 3808 cni.go:473] [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:6d:c3:a0:bc:6b\",\"name\":\"69a37657c825003\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b68ad1f5-9852-4f20-88ee-5aa207273bd5\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:20:20.162562 3808 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0421 03:20:23.290517 3808 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:23.291072 3808 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:23.310083 3808 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:23.310101 3808 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:23.316877 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0421 03:20:23.316902 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0421 03:20:24.054795 3808 namespace.go:98] [integration-service] adding namespace I0421 03:20:24.054896 3808 namespace.go:98] [default-tenant] adding namespace I0421 03:20:24.055720 3808 namespace.go:102] [integration-service] adding namespace took 903.612µs I0421 03:20:24.056325 3808 namespace.go:102] [default-tenant] adding namespace took 1.411003ms I0421 03:20:24.060055 3808 namespace.go:98] [build-service] adding namespace I0421 03:20:24.060832 3808 namespace.go:102] [build-service] adding namespace took 757.964µs I0421 03:20:24.060977 3808 namespace.go:98] [image-controller] adding namespace I0421 03:20:24.061658 3808 namespace.go:102] [image-controller] adding namespace took 666.242µs I0421 03:20:24.110244 3808 namespace.go:142] [integration-service] updating namespace I0421 03:20:24.127322 3808 namespace.go:142] [integration-service] updating namespace I0421 03:20:24.133323 3808 namespace.go:142] [default-tenant] updating namespace I0421 03:20:24.160338 3808 namespace.go:142] [default-tenant] updating namespace I0421 03:20:24.178344 3808 namespace.go:142] [build-service] updating namespace I0421 03:20:24.198077 3808 namespace.go:142] [build-service] updating namespace I0421 03:20:24.227948 3808 namespace.go:142] [image-controller] updating namespace I0421 03:20:24.268974 3808 namespace.go:142] [image-controller] updating namespace I0421 03:20:24.411016 3808 namespace.go:142] [build-service] updating namespace I0421 03:20:24.434453 3808 namespace.go:142] [integration-service] updating namespace I0421 03:20:24.459102 3808 namespace.go:98] [release-service] adding namespace I0421 03:20:24.459960 3808 namespace.go:102] [release-service] adding namespace took 838.279µs I0421 03:20:24.483895 3808 namespace.go:142] [release-service] updating namespace I0421 03:20:24.521038 3808 namespace.go:142] [release-service] updating namespace I0421 03:20:24.530302 3808 namespace.go:142] [konflux-ui] updating namespace I0421 03:20:24.972990 3808 namespace.go:98] [konflux-info] adding namespace I0421 03:20:24.973916 3808 namespace.go:102] [konflux-info] adding namespace took 903.64µs I0421 03:20:25.019921 3808 namespace.go:142] [konflux-info] updating namespace I0421 03:20:25.020015 3808 namespace.go:142] [konflux-info] updating namespace I0421 03:20:25.100046 3808 namespace.go:98] [namespace-lister] adding namespace I0421 03:20:25.100680 3808 namespace.go:102] [namespace-lister] adding namespace took 617.382µs I0421 03:20:25.151043 3808 namespace.go:142] [namespace-lister] updating namespace I0421 03:20:25.166221 3808 cni.go:452] [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key ] I0421 03:20:25.193334 3808 namespace.go:142] [namespace-lister] updating namespace I0421 03:20:25.262778 3808 cni.go:473] [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/tuf-repository-init-9j75t-rdbk4 69a37657c825003f860242c1842edc825b54ffbc388edca80408b0fbbc1da9c3 network default NAD default NAD key default], result "{}", err I0421 03:20:25.382588 3808 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-9j75t-rdbk4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:25.382619 3808 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-9j75t-rdbk4 I0421 03:20:25.383927 3808 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-9j75t-rdbk4, ips: 10.128.0.44 I0421 03:20:25.383956 3808 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-9j75t-rdbk4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:25.418302 3808 namespace.go:98] [enterprise-contract-service] adding namespace I0421 03:20:25.419050 3808 namespace.go:102] [enterprise-contract-service] adding namespace took 726.203µs I0421 03:20:25.419086 3808 namespace.go:142] [enterprise-contract-service] updating namespace I0421 03:20:25.471074 3808 namespace.go:142] [enterprise-contract-service] updating namespace I0421 03:20:25.546207 3808 namespace.go:142] [image-controller] updating namespace I0421 03:20:25.824205 3808 namespace.go:142] [default-tenant] updating namespace I0421 03:20:25.994218 3808 namespace.go:98] [segment-bridge] adding namespace I0421 03:20:25.995030 3808 namespace.go:102] [segment-bridge] adding namespace took 795.132µs I0421 03:20:26.265497 3808 namespace.go:142] [segment-bridge] updating namespace I0421 03:20:26.305569 3808 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0421 03:20:26.306348 3808 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0421 03:20:26.306374 3808 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0421 03:20:26.309324 3808 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0421 03:20:26.407160 3808 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0421 03:20:26.407184 3808 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0421 03:20:26.407940 3808 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0421 03:20:26.502666 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=127 I0421 03:20:26.544220 3808 namespace.go:142] [segment-bridge] updating namespace I0421 03:20:26.672440 3808 namespace.go:142] [build-service] updating namespace I0421 03:20:26.678993 3808 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-9j75t-rdbk4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:26.679010 3808 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-9j75t-rdbk4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:26.862030 3808 base_network_controller_pods.go:487] [default/konflux-ui/dex-7bdcc957b5-5d6kh] creating logical port konflux-ui_dex-7bdcc957b5-5d6kh for pod on switch ip-10-0-2-88.ec2.internal I0421 03:20:26.862170 3808 kube.go:256] Updating pod konflux-ui/dex-7bdcc957b5-5d6kh I0421 03:20:26.883242 3808 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-xc7rt] creating logical port namespace-lister_namespace-lister-67655c4674-xc7rt for pod on switch ip-10-0-2-88.ec2.internal I0421 03:20:26.883369 3808 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-xc7rt I0421 03:20:26.926600 3808 pod.go:62] [konflux-ui/dex-7bdcc957b5-5d6kh] pod update took 64.47202ms I0421 03:20:26.926622 3808 base_network_controller_pods.go:951] [default/konflux-ui/dex-7bdcc957b5-5d6kh] addLogicalPort annotation time took 64.498628ms I0421 03:20:26.928056 3808 pods.go:271] [konflux-ui/dex-7bdcc957b5-5d6kh] addLogicalPort took 66.043035ms, libovsdb time 991.509µs I0421 03:20:26.928128 3808 base_network_controller_pods.go:487] [default/konflux-ui/proxy-7fc7b59b75-6prkl] creating logical port konflux-ui_proxy-7fc7b59b75-6prkl for pod on switch ip-10-0-2-88.ec2.internal I0421 03:20:26.928228 3808 kube.go:256] Updating pod konflux-ui/proxy-7fc7b59b75-6prkl I0421 03:20:26.946351 3808 pod.go:62] [namespace-lister/namespace-lister-67655c4674-xc7rt] pod update took 63.005999ms I0421 03:20:26.946375 3808 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-67655c4674-xc7rt] addLogicalPort annotation time took 63.033438ms I0421 03:20:26.947643 3808 pods.go:271] [namespace-lister/namespace-lister-67655c4674-xc7rt] addLogicalPort took 64.421224ms, libovsdb time 908.454µs I0421 03:20:26.969407 3808 pod.go:62] [konflux-ui/proxy-7fc7b59b75-6prkl] pod update took 41.207852ms I0421 03:20:26.969428 3808 base_network_controller_pods.go:951] [default/konflux-ui/proxy-7fc7b59b75-6prkl] addLogicalPort annotation time took 41.23243ms I0421 03:20:26.970568 3808 pods.go:271] [konflux-ui/proxy-7fc7b59b75-6prkl] addLogicalPort took 42.44581ms, libovsdb time 791.173µs I0421 03:20:27.047270 3808 namespace.go:142] [default-tenant] updating namespace I0421 03:20:27.468396 3808 namespace.go:142] [image-controller] updating namespace I0421 03:20:27.691605 3808 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-mk9lk] creating logical port tsf-tas_tuf-bc5768c88-mk9lk for pod on switch ip-10-0-2-88.ec2.internal I0421 03:20:27.691731 3808 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-mk9lk I0421 03:20:27.728324 3808 pod.go:62] [tsf-tas/tuf-bc5768c88-mk9lk] pod update took 36.613ms I0421 03:20:27.728345 3808 base_network_controller_pods.go:951] [default/tsf-tas/tuf-bc5768c88-mk9lk] addLogicalPort annotation time took 36.63952ms I0421 03:20:27.729578 3808 pods.go:271] [tsf-tas/tuf-bc5768c88-mk9lk] addLogicalPort took 37.988563ms, libovsdb time 749.63µs I0421 03:20:27.990885 3808 namespace.go:142] [integration-service] updating namespace I0421 03:20:28.436189 3808 namespace.go:142] [konflux-info] updating namespace I0421 03:20:28.836235 3808 namespace.go:142] [release-service] updating namespace I0421 03:20:29.002339 3808 namespace.go:142] [build-service] updating namespace I0421 03:20:29.051348 3808 namespace.go:142] [default-tenant] updating namespace I0421 03:20:29.067244 3808 namespace.go:142] [image-controller] updating namespace I0421 03:20:29.114252 3808 namespace.go:142] [integration-service] updating namespace I0421 03:20:29.141682 3808 namespace.go:142] [konflux-info] updating namespace I0421 03:20:29.164254 3808 namespace.go:142] [release-service] updating namespace I0421 03:20:29.290332 3808 namespace.go:142] [build-service] updating namespace I0421 03:20:29.401299 3808 namespace.go:142] [default-tenant] updating namespace I0421 03:20:29.502346 3808 namespace.go:142] [image-controller] updating namespace I0421 03:20:29.554753 3808 cni.go:452] [tsf-tas/tuf-bc5768c88-mk9lk 6995fcee795dec817b491d394054001c919b1645b4113405857438ae2a145307 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-bc5768c88-mk9lk 6995fcee795dec817b491d394054001c919b1645b4113405857438ae2a145307 network default NAD default NAD key ] I0421 03:20:29.564633 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-mk9lk, hostIfaceName: 6995fcee795dec8, network: default, NAD default, SandboxID: "6995fcee795dec817b491d394054001c919b1645b4113405857438ae2a145307", PCI device ID: , UID: "784e9629-c274-4d02-a25e-d73541a2c390", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0421 03:20:29.574368 3808 namespace.go:142] [integration-service] updating namespace I0421 03:20:29.621211 3808 cni.go:473] [tsf-tas/tuf-bc5768c88-mk9lk 6995fcee795dec817b491d394054001c919b1645b4113405857438ae2a145307 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-mk9lk 6995fcee795dec817b491d394054001c919b1645b4113405857438ae2a145307 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:d3:b0:10:2c:c4\",\"name\":\"6995fcee795dec8\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18e759f1-5336-4f43-99b2-2c10906ce8ed\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:20:29.672683 3808 namespace.go:142] [konflux-info] updating namespace I0421 03:20:29.763616 3808 namespace.go:142] [release-service] updating namespace I0421 03:20:30.969585 3808 namespace.go:142] [enterprise-contract-service] updating namespace I0421 03:20:31.137479 3808 namespace.go:142] [namespace-lister] updating namespace I0421 03:20:31.269537 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0421 03:20:31.289211 3808 namespace.go:142] [segment-bridge] updating namespace I0421 03:20:31.418321 3808 namespace.go:142] [enterprise-contract-service] updating namespace I0421 03:20:31.437223 3808 namespace.go:142] [namespace-lister] updating namespace I0421 03:20:31.457123 3808 namespace.go:142] [segment-bridge] updating namespace I0421 03:20:31.569535 3808 namespace.go:142] [enterprise-contract-service] updating namespace I0421 03:20:31.660048 3808 namespace.go:142] [namespace-lister] updating namespace I0421 03:20:31.763028 3808 namespace.go:142] [segment-bridge] updating namespace W0421 03:20:32.994072 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0421 03:20:33.019598 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0421 03:20:33.207643 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0421 03:20:34.740710 3808 cni.go:452] [konflux-ui/dex-7bdcc957b5-5d6kh 44f43f99e4000419ab7a58a86ce87de4a0d064dbd3595d75e808ac112fef076e network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-7bdcc957b5-5d6kh 44f43f99e4000419ab7a58a86ce87de4a0d064dbd3595d75e808ac112fef076e network default NAD default NAD key ] I0421 03:20:34.747017 3808 cni.go:452] [namespace-lister/namespace-lister-67655c4674-xc7rt be71594bb98dbf88cddd87a6bcc2bc1e04c70740878dbd533e953b9913a39e83 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-xc7rt be71594bb98dbf88cddd87a6bcc2bc1e04c70740878dbd533e953b9913a39e83 network default NAD default NAD key ] I0421 03:20:34.750957 3808 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-7bdcc957b5-5d6kh, hostIfaceName: 44f43f99e400041, network: default, NAD default, SandboxID: "44f43f99e4000419ab7a58a86ce87de4a0d064dbd3595d75e808ac112fef076e", PCI device ID: , UID: "4f79ec69-67d3-41ed-8f4b-0c56b21de2b9", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0421 03:20:34.757186 3808 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-xc7rt, hostIfaceName: be71594bb98dbf8, network: default, NAD default, SandboxID: "be71594bb98dbf88cddd87a6bcc2bc1e04c70740878dbd533e953b9913a39e83", PCI device ID: , UID: "18645374-99f2-4dfc-b699-0b1ede2e73f5", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0421 03:20:34.807570 3808 cni.go:473] [konflux-ui/dex-7bdcc957b5-5d6kh 44f43f99e4000419ab7a58a86ce87de4a0d064dbd3595d75e808ac112fef076e network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-7bdcc957b5-5d6kh 44f43f99e4000419ab7a58a86ce87de4a0d064dbd3595d75e808ac112fef076e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:51:6f:2e:d8:d1\",\"name\":\"44f43f99e400041\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ad1626d-e340-49b8-b62c-98ea13e43469\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:20:34.833018 3808 cni.go:473] [namespace-lister/namespace-lister-67655c4674-xc7rt be71594bb98dbf88cddd87a6bcc2bc1e04c70740878dbd533e953b9913a39e83 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-xc7rt be71594bb98dbf88cddd87a6bcc2bc1e04c70740878dbd533e953b9913a39e83 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:02:14:bb:fa:17\",\"name\":\"be71594bb98dbf8\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c01fdd7-347a-40f9-852b-e7377a7b0cde\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:20:36.289170 3808 cni.go:452] [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key ] I0421 03:20:36.303274 3808 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-7fc7b59b75-6prkl, hostIfaceName: 5c77164b27fa42f, network: default, NAD default, SandboxID: "5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe", PCI device ID: , UID: "74d4b60e-bcb4-4396-a020-4c9572415e58", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0421 03:20:36.357383 3808 cni.go:473] [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:65:7c:52:e2:7d\",\"name\":\"5c77164b27fa42f\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/caffd91b-7f43-43c2-acab-1408c45a72ab\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0421 03:20:37.239402 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0421 03:20:37.258451 3808 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0421 03:20:38.269694 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister W0421 03:20:41.295047 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0421 03:20:43.299462 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0421 03:20:43.299483 3808 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0421 03:20:44.079542 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0421 03:20:44.079563 3808 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0421 03:20:44.108438 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0421 03:20:44.111549 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0421 03:20:44.136536 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0421 03:20:44.139377 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0421 03:20:44.166907 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0421 03:20:44.174055 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0421 03:20:44.224233 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0421 03:20:53.494474 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=142 I0421 03:20:55.411480 3808 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-m7j7z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:55.411986 3808 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-m7j7z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:20:56.149303 3808 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-m7j7z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:56.149326 3808 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-m7j7z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:20:56.154590 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-m7j7z I0421 03:20:56.154617 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-m7j7z I0421 03:21:11.918182 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 W0421 03:21:23.360872 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0421 03:21:23.360894 3808 util.go:959] Failed to build node endpoints for node ip-10-0-2-88.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy I0421 03:21:26.228417 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0421 03:21:43.503680 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=609 I0421 03:21:50.115507 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0421 03:21:54.198629 3808 cni.go:452] [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key ] I0421 03:21:54.278600 3808 cni.go:473] [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-7fc7b59b75-6prkl 5c77164b27fa42f8c89f2621c26772e7d5c778423735e330af649f537b33babe network default NAD default NAD key default], result "{}", err I0421 03:21:54.339503 3808 obj_retry.go:498] Detected object konflux-ui/proxy-7fc7b59b75-6prkl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:21:54.339533 3808 pods.go:176] Deleting pod: konflux-ui/proxy-7fc7b59b75-6prkl I0421 03:21:54.340815 3808 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-7fc7b59b75-6prkl, ips: 10.128.0.47 I0421 03:21:54.340840 3808 obj_retry.go:498] Detected object konflux-ui/proxy-7fc7b59b75-6prkl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:21:54.669793 3808 obj_retry.go:492] Detected object konflux-ui/proxy-7fc7b59b75-6prkl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:21:54.669822 3808 obj_retry.go:492] Detected object konflux-ui/proxy-7fc7b59b75-6prkl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:21:54.676318 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-7fc7b59b75-6prkl I0421 03:21:54.676346 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-7fc7b59b75-6prkl I0421 03:21:57.665074 3808 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:21:57.665609 3808 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:21:57.707622 3808 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:21:57.707640 3808 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:21:57.713886 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0421 03:21:57.713908 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0421 03:22:26.966282 3808 namespace.go:98] [default-managed-tenant-cled5] adding namespace I0421 03:22:26.967342 3808 namespace.go:102] [default-managed-tenant-cled5] adding namespace took 1.033085ms I0421 03:22:26.994936 3808 namespace.go:142] [default-managed-tenant-cled5] updating namespace I0421 03:22:27.014547 3808 namespace.go:142] [default-managed-tenant-cled5] updating namespace I0421 03:22:27.969871 3808 namespace.go:142] [default-managed-tenant-cled5] updating namespace I0421 03:22:28.102810 3808 namespace.go:142] [default-managed-tenant-cled5] updating namespace I0421 03:22:28.202284 3808 namespace.go:142] [default-managed-tenant-cled5] updating namespace I0421 03:22:32.603877 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0421 03:23:08.811948 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0421 03:23:41.630810 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0421 03:23:53.496241 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=74 I0421 03:24:04.495584 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0421 03:24:12.145209 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0421 03:24:32.031506 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0421 03:26:59.495847 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=56 I0421 03:27:11.708504 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0421 03:27:36.119547 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0421 03:28:17.527464 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=42 I0421 03:28:27.507828 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=16 I0421 03:28:36.500372 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=23 I0421 03:29:10.922611 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0421 03:29:28.274221 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0421 03:29:32.611796 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0421 03:29:47.232687 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0421 03:30:00.212109 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29612370-m2cqv for pod on switch ip-10-0-2-88.ec2.internal I0421 03:30:00.212245 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv I0421 03:30:00.241553 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv] pod update took 29.333659ms I0421 03:30:00.241574 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv] addLogicalPort annotation time took 29.360002ms I0421 03:30:00.242771 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv] addLogicalPort took 30.681758ms, libovsdb time 833.176µs I0421 03:30:00.573501 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key ] I0421 03:30:00.583207 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29612370-m2cqv, hostIfaceName: 34a8d3fec63692e, network: default, NAD default, SandboxID: "34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d", PCI device ID: , UID: "4c821bf4-e05a-4e49-8f8f-a1f374e9b652", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0421 03:30:00.636561 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:13:19:0d:9f:f7\",\"name\":\"34a8d3fec63692e\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/88a61cfc-7eb4-41d1-9409-77d645e96ea9\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:30:04.331021 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key ] I0421 03:30:04.426027 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv 34a8d3fec63692ef147dd3f33d8dd399fb36457a93e1cb6619a5570a3a35ff5d network default NAD default NAD key default], result "{}", err I0421 03:30:04.474664 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:30:04.474719 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv I0421 03:30:04.476035 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv, ips: 10.128.0.49 I0421 03:30:04.476057 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:30:05.058487 3808 obj_retry.go:492] Detected object tsf-tpa/create-db-f6pgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:05.058541 3808 obj_retry.go:492] Detected object tsf-tpa/create-db-f6pgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:05.064127 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-f6pgz I0421 03:30:05.064172 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-f6pgz I0421 03:30:05.349115 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:05.349175 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612370-m2cqv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:05.377692 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:05.377710 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:05.380409 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l I0421 03:30:05.380441 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612340-8dq7l I0421 03:30:07.035399 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0421 03:30:08.029937 3808 obj_retry.go:492] Detected object tsf-tpa/migrate-db-bfpfm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:08.029957 3808 obj_retry.go:492] Detected object tsf-tpa/migrate-db-bfpfm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:08.032423 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-bfpfm I0421 03:30:08.032448 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-bfpfm I0421 03:30:15.032643 3808 obj_retry.go:492] Detected object tsf-tpa/create-importers-dhltm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:15.032685 3808 obj_retry.go:492] Detected object tsf-tpa/create-importers-dhltm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:30:15.037361 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-dhltm I0421 03:30:15.037382 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-dhltm I0421 03:31:34.816128 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0421 03:31:57.499868 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0421 03:33:09.500790 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0421 03:33:16.148790 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0421 03:33:33.635348 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0421 03:34:02.512110 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=23 I0421 03:34:22.712631 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0421 03:35:01.926476 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0421 03:35:25.499642 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=51 I0421 03:36:12.123912 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 W0421 03:36:20.852431 3808 duration.go:461] Config duration recorder: measurement expired for pod/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2v6gt I0421 03:36:45.898384 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-pull-request-p9q64-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:36:45.898964 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-pull-request-p9q64-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:36:46.649118 3808 obj_retry.go:492] Detected object default-tenant/tsf-demo-comp-on-pull-request-p9q64-init-pod of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:36:46.649183 3808 obj_retry.go:492] Detected object default-tenant/tsf-demo-comp-on-pull-request-p9q64-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:36:46.657817 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant/tsf-demo-comp-on-pull-request-p9q64-init-pod I0421 03:36:46.657840 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod default-tenant/tsf-demo-comp-on-pull-request-p9q64-init-pod I0421 03:36:48.822737 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:36:48.823326 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:37:05.990909 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:37:05.991488 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:37:06.334741 3808 base_network_controller_pods.go:487] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod] creating logical port default-tenant_tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod for pod on switch ip-10-0-2-88.ec2.internal I0421 03:37:06.334896 3808 kube.go:256] Updating pod default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod I0421 03:37:06.360650 3808 pod.go:62] [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod] pod update took 25.796649ms I0421 03:37:06.360674 3808 base_network_controller_pods.go:951] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod] addLogicalPort annotation time took 25.826306ms I0421 03:37:06.361993 3808 pods.go:271] [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod] addLogicalPort took 27.271335ms, libovsdb time 928.664µs I0421 03:37:06.702712 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key ] ADD starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key ] I0421 03:37:06.712917 3808 helper_linux.go:556] ConfigureOVS: namespace: default-tenant, podName: tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod, hostIfaceName: fcf18ee6757051e, network: default, NAD default, SandboxID: "fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac", PCI device ID: , UID: "90c8a212-ae8b-4ddf-945d-dd0fa70eff8f", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0421 03:37:06.767807 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key default] ADD finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:57:c3:bc:b9:f1\",\"name\":\"fcf18ee6757051e\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dafc7660-69be-41b9-a4e2-9d527b7ac1af\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:37:48.867347 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key ] DEL starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key ] I0421 03:37:48.963156 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key default] DEL finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod fcf18ee6757051eafc36f131ad98ad8325bbe32e306870e69f29d2b2d289f4ac network default NAD default NAD key default], result "{}", err I0421 03:37:49.074496 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:37:49.074528 3808 pods.go:176] Deleting pod: default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod I0421 03:37:49.075900 3808 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod, ips: 10.128.0.50 I0421 03:37:49.075929 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:37:52.531890 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0421 03:38:26.616816 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0421 03:38:30.504565 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=13 I0421 03:38:41.280594 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0421 03:38:46.039735 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0421 03:39:05.851860 3808 duration.go:461] Config duration recorder: measurement expired for pod/tsf-tas/segment-backup-installation-tkgl7-69l5d I0421 03:39:08.239069 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0421 03:40:00.199396 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29612380-dzjj4 for pod on switch ip-10-0-2-88.ec2.internal I0421 03:40:00.199530 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 I0421 03:40:00.226639 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4] pod update took 27.134369ms I0421 03:40:00.226663 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4] addLogicalPort annotation time took 27.162502ms I0421 03:40:00.228104 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4] addLogicalPort took 28.724361ms, libovsdb time 1.024963ms I0421 03:40:00.560819 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key ] I0421 03:40:00.570858 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29612380-dzjj4, hostIfaceName: e0f3cd91bceadde, network: default, NAD default, SandboxID: "e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71", PCI device ID: , UID: "1e88731e-be63-45ed-9080-b61a715b87cf", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0421 03:40:00.623991 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:f0:d8:23:96:3e\",\"name\":\"e0f3cd91bceadde\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67c4921d-8875-46f3-ab56-b752ffa04600\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:40:03.628217 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key ] I0421 03:40:03.721338 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 e0f3cd91bceaddec14c82e2992af9f9214b3be3b3f23c0ab2dd235c34495ca71 network default NAD default NAD key default], result "{}", err I0421 03:40:03.769056 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:40:03.769107 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 I0421 03:40:03.770517 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4, ips: 10.128.0.51 I0421 03:40:03.770546 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:40:04.640187 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:40:04.640215 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612380-dzjj4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:40:04.678629 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:40:04.678650 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:40:04.682043 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 I0421 03:40:04.682068 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29612350-mtpl7 I0421 03:40:16.503726 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0421 03:40:27.662960 3808 base_network_controller_pods.go:487] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod] creating logical port default-tenant_tsf-demo-comp-on-push-j6v8t-build-image-index-pod for pod on switch ip-10-0-2-88.ec2.internal I0421 03:40:27.663128 3808 kube.go:256] Updating pod default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod I0421 03:40:27.697477 3808 pod.go:62] [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod] pod update took 34.383808ms I0421 03:40:27.697499 3808 base_network_controller_pods.go:951] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod] addLogicalPort annotation time took 34.409439ms I0421 03:40:27.698785 3808 pods.go:271] [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod] addLogicalPort took 35.840431ms, libovsdb time 908.956µs I0421 03:40:28.025392 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key ] ADD starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key ] I0421 03:40:28.034933 3808 helper_linux.go:556] ConfigureOVS: namespace: default-tenant, podName: tsf-demo-comp-on-push-j6v8t-build-image-index-pod, hostIfaceName: 95f633e30716b2f, network: default, NAD default, SandboxID: "95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6", PCI device ID: , UID: "7889ee2b-792a-47ac-a0f3-664e2df96207", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0421 03:40:28.090818 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key default] ADD finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:0f:aa:48:86:94\",\"name\":\"95f633e30716b2f\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6de49d08-dca5-4b0a-96db-f37237cc5c66\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:40:28.336609 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:40:28.337252 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:40:46.820793 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0421 03:40:50.153664 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0421 03:41:10.806175 3808 base_network_controller_pods.go:487] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod] creating logical port default-tenant_tsf-demo-comp-on-push-j6v8t-clamav-scan-pod for pod on switch ip-10-0-2-88.ec2.internal I0421 03:41:10.806291 3808 kube.go:256] Updating pod default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod I0421 03:41:10.832287 3808 pod.go:62] [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod] pod update took 26.029833ms I0421 03:41:10.832315 3808 base_network_controller_pods.go:951] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod] addLogicalPort annotation time took 26.059589ms I0421 03:41:10.833672 3808 pods.go:271] [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod] addLogicalPort took 27.513023ms, libovsdb time 902.404µs I0421 03:41:11.048994 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key ] DEL starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key ] I0421 03:41:11.139675 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key default] DEL finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod 95f633e30716b2f4380fc568e6194e0209774d4a04e8e1979fcc9795cd48d6a6 network default NAD default NAD key default], result "{}", err I0421 03:41:11.244952 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key ] ADD starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key ] I0421 03:41:11.254954 3808 helper_linux.go:556] ConfigureOVS: namespace: default-tenant, podName: tsf-demo-comp-on-push-j6v8t-clamav-scan-pod, hostIfaceName: 5475b2041a1f245, network: default, NAD default, SandboxID: "5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391", PCI device ID: , UID: "fd171a56-e24c-405a-842d-f77e8a12d2a4", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0421 03:41:11.264473 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:41:11.264579 3808 pods.go:176] Deleting pod: default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod I0421 03:41:11.266129 3808 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod, ips: 10.128.0.52 I0421 03:41:11.266189 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:41:11.311819 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key default] ADD finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:a8:b6:67:71:1f\",\"name\":\"5475b2041a1f245\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0268c13-faca-4494-9a15-39b708033055\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:41:12.542309 3808 base_network_controller_pods.go:487] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod] creating logical port default-tenant_tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod for pod on switch ip-10-0-2-88.ec2.internal I0421 03:41:12.542431 3808 kube.go:256] Updating pod default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod I0421 03:41:12.574779 3808 pod.go:62] [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod] pod update took 32.377661ms I0421 03:41:12.574799 3808 base_network_controller_pods.go:951] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod] addLogicalPort annotation time took 32.411639ms I0421 03:41:12.576047 3808 pods.go:271] [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod] addLogicalPort took 33.751446ms, libovsdb time 812.986µs I0421 03:41:12.664525 3808 base_network_controller_pods.go:487] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod] creating logical port default-tenant_tsf-demo-comp-on-push-j6v8t-tpa-scan-pod for pod on switch ip-10-0-2-88.ec2.internal I0421 03:41:12.664679 3808 kube.go:256] Updating pod default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod I0421 03:41:12.699276 3808 pod.go:62] [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod] pod update took 34.630867ms I0421 03:41:12.699302 3808 base_network_controller_pods.go:951] [default/default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod] addLogicalPort annotation time took 34.660272ms I0421 03:41:12.700674 3808 pods.go:271] [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod] addLogicalPort took 36.165653ms, libovsdb time 932.339µs I0421 03:41:12.948164 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key ] ADD starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key ] I0421 03:41:12.958912 3808 helper_linux.go:556] ConfigureOVS: namespace: default-tenant, podName: tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod, hostIfaceName: 5a30de9f74cbd49, network: default, NAD default, SandboxID: "5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c", PCI device ID: , UID: "1f1df600-b241-4973-b18a-be5bde66c64e", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0421 03:41:13.011227 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key default] ADD finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:d4:46:ff:2d:32\",\"name\":\"5a30de9f74cbd49\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/48ed18f3-cc6c-4ec1-a207-c7a8dd4e8178\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:41:13.069188 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key ] ADD starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key ] I0421 03:41:13.080457 3808 helper_linux.go:556] ConfigureOVS: namespace: default-tenant, podName: tsf-demo-comp-on-push-j6v8t-tpa-scan-pod, hostIfaceName: 08d73d7f7bcd536, network: default, NAD default, SandboxID: "08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3", PCI device ID: , UID: "0c705dc7-0f4d-442b-b713-34a536fedbd2", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0421 03:41:13.134481 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key default] ADD finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:16:19:58:62:65\",\"name\":\"08d73d7f7bcd536\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5217c6f9-985e-4ebb-95ba-3b2358ab72ab\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:41:22.504683 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0421 03:41:45.639130 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0421 03:41:55.127991 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0421 03:41:58.930668 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0421 03:42:10.313266 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:10.313939 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:13.199077 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:13.199670 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:14.215206 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:14.215701 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:18.552786 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key ] DEL starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key ] I0421 03:42:18.654282 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key default] DEL finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod 08d73d7f7bcd5360bf71b0e638a33ef6121d0ff29da8facfe1a11496b6bd73f3 network default NAD default NAD key default], result "{}", err I0421 03:42:18.757028 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:18.757065 3808 pods.go:176] Deleting pod: default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod I0421 03:42:18.758383 3808 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod, ips: 10.128.0.55 I0421 03:42:18.758406 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-tpa-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:28.515975 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=186 I0421 03:42:32.648804 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key ] DEL starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key ] I0421 03:42:32.739172 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key default] DEL finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod 5a30de9f74cbd491bf60daff1c12a1fc4e91b9ee215f59b9a61d40d0d44d623c network default NAD default NAD key default], result "{}", err I0421 03:42:32.901822 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:32.901857 3808 pods.go:176] Deleting pod: default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod I0421 03:42:32.903243 3808 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod, ips: 10.128.0.54 I0421 03:42:32.903280 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:44.412463 3808 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5rzdh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:44.413033 3808 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5rzdh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:45.129051 3808 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5rzdh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:42:45.129072 3808 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5rzdh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:42:45.135578 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-5rzdh I0421 03:42:45.135599 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-5rzdh I0421 03:42:59.502071 3808 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2d8sp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:42:59.502644 3808 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2d8sp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:43:00.364917 3808 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2d8sp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:43:00.364941 3808 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2d8sp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0421 03:43:00.370587 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-2d8sp I0421 03:43:00.370611 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-2d8sp I0421 03:43:02.716424 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0421 03:43:40.503384 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=44 I0421 03:44:34.284933 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0421 03:45:06.243824 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0421 03:45:08.449766 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:08.450352 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:16.678314 3808 cni.go:452] [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key ] DEL starting CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key ] I0421 03:45:16.761744 3808 cni.go:473] [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key default] DEL finished CNI request [default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod 5475b2041a1f245c987efd48a66724b768968078cef3bd596d53536b1caf0391 network default NAD default NAD key default], result "{}", err I0421 03:45:16.829015 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:16.829042 3808 pods.go:176] Deleting pod: default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod I0421 03:45:16.830419 3808 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod, ips: 10.128.0.53 I0421 03:45:16.830454 3808 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-j6v8t-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:32.044211 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0421 03:45:33.769982 3808 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-plwk-kt92g-verify-pod] creating logical port default-tenant_my-integration-test-plwk-kt92g-verify-pod for pod on switch ip-10-0-2-88.ec2.internal I0421 03:45:33.770124 3808 kube.go:256] Updating pod default-tenant/my-integration-test-plwk-kt92g-verify-pod I0421 03:45:33.811453 3808 pod.go:62] [default-tenant/my-integration-test-plwk-kt92g-verify-pod] pod update took 41.376589ms I0421 03:45:33.811482 3808 base_network_controller_pods.go:951] [default/default-tenant/my-integration-test-plwk-kt92g-verify-pod] addLogicalPort annotation time took 41.408958ms I0421 03:45:33.812776 3808 pods.go:271] [default-tenant/my-integration-test-plwk-kt92g-verify-pod] addLogicalPort took 42.811072ms, libovsdb time 892.665µs I0421 03:45:34.139051 3808 cni.go:452] [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key ] ADD starting CNI request [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key ] I0421 03:45:34.148508 3808 helper_linux.go:556] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-plwk-kt92g-verify-pod, hostIfaceName: 7ba2051cf4d8895, network: default, NAD default, SandboxID: "7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae", PCI device ID: , UID: "430c4ddf-e7c9-425f-a8fd-f30dab70f1ec", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0421 03:45:34.203563 3808 cni.go:473] [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key default] ADD finished CNI request [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:e9:33:60:10:c3\",\"name\":\"7ba2051cf4d8895\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c9cfa4b9-0292-42c7-af6c-025fbc74595e\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0421 03:45:34.683075 3808 obj_retry.go:498] Detected object default-tenant/my-integration-test-plwk-kt92g-collect-keyless-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:34.683651 3808 obj_retry.go:498] Detected object default-tenant/my-integration-test-plwk-kt92g-collect-keyless-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:58.988534 3808 cni.go:452] [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key ] DEL starting CNI request [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key ] I0421 03:45:59.087567 3808 cni.go:473] [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key default] DEL finished CNI request [default-tenant/my-integration-test-plwk-kt92g-verify-pod 7ba2051cf4d88950f1682035744fec4918fb40e0c9f057831085458d708e3cae network default NAD default NAD key default], result "{}", err I0421 03:45:59.245202 3808 obj_retry.go:498] Detected object default-tenant/my-integration-test-plwk-kt92g-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0421 03:45:59.245238 3808 pods.go:176] Deleting pod: default-tenant/my-integration-test-plwk-kt92g-verify-pod I0421 03:45:59.246703 3808 pods.go:236] Attempting to release IPs for pod: default-tenant/my-integration-test-plwk-kt92g-verify-pod, ips: 10.128.0.56 I0421 03:45:59.246736 3808 obj_retry.go:498] Detected object default-tenant/my-integration-test-plwk-kt92g-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it