+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-90.ec2.internal ++ [[ -n ip-10-0-0-90.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-90.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0326 23:12:44.315474186 - disable conntrack on geneve port' I0326 23:12:44.315474186 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0326 23:12:44.332384445 - starting ovnkube-node + echo 'I0326 23:12:44.332384445 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-90.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-90.ec2.internal --init-node ip-10-0-0-90.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-90.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0326 23:12:44.375904 3952 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0326 23:12:44.375948 3952 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-7a3fdde624.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0326 23:12:44.377688 3952 kube.go:419] Waiting for certificate I0326 23:12:44.377726 3952 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0326 23:12:44.377781 3952 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0326 23:12:44.378566 3952 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:12:44.409436 3952 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:12:44.409453 3952 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:12:44.412431 3952 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:12:44.412535 3952 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-r64xz" I0326 23:12:48.052304 3952 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-r64xz" I0326 23:12:48.052424 3952 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:12:48.377831 3952 kube.go:426] Certificate found I0326 23:12:48.378395 3952 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:12:48.379516 3952 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0326 23:12:48.456714 3952 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:12:48.465572 3952 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0326 23:12:48.466213 3952 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0326 23:12:48.466298 3952 factory.go:531] Starting watch factory I0326 23:12:48.466454 3952 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466469 3952 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466478 3952 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466480 3952 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466498 3952 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466514 3952 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466565 3952 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466563 3952 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466484 3952 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.466594 3952 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.467309 3952 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.467499 3952 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.475438 3952 controller_manager.go:368] Starting the ovnkube controller I0326 23:12:48.475458 3952 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-90.ec2.internal I0326 23:12:48.475541 3952 controller_manager.go:393] NBDB zone sync took: 72.586µs I0326 23:12:48.475556 3952 factory.go:531] Starting watch factory I0326 23:12:48.545160 3952 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.545343 3952 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.545448 3952 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.545720 3952 ovn_northd.go:121] Registering OVN northd metrics I0326 23:12:48.545758 3952 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.545884 3952 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0326 23:12:48.545997 3952 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.553429 3952 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0326 23:12:48.553469 3952 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0326 23:12:48.562831 3952 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:12:48.570790 3952 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0326 23:12:48.573272 3952 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:12:48.573288 3952 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:12:48.573292 3952 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:12:48.573295 3952 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:12:48.573298 3952 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:12:48.573301 3952 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:12:48.573365 3952 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:12:48.573378 3952 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:12:48.573386 3952 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:12:48.573402 3952 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:12:48.581782 3952 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:12:48.581794 3952 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:12:48.581798 3952 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:12:48.581801 3952 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:12:48.581804 3952 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:12:48.581807 3952 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:12:48.589822 3952 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:12:48.589828 3952 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:12:48.679584 3952 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:12:48.679608 3952 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:12:48.679702 3952 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.679719 3952 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.687177 3952 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:12:48.687196 3952 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:12:48.694177 3952 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.785163 3952 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:12:48.785266 3952 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.785283 3952 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.793280 3952 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:12:48.796798 3952 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.798686 3952 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:12:48.798752 3952 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.798765 3952 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.811382 3952 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.891812 3952 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:12:48.896985 3952 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:12:48.897063 3952 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.897077 3952 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:48.905637 3952 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:12:48.914562 3952 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.003053 3952 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:12:49.003134 3952 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.003154 3952 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.010164 3952 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.010931 3952 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:12:49.016602 3952 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:12:49.016854 3952 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:12:49.016869 3952 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:12:49.020837 3952 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:12:49.021709 3952 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:12:49.021792 3952 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.021806 3952 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.021842 3952 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.021856 3952 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.025457 3952 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.025487 3952 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:12:49.027474 3952 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:12:49.027488 3952 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:12:49.027495 3952 factory.go:636] Watch Factory start up complete, took: 551.938879ms I0326 23:12:49.027502 3952 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-90.ec2.internal" zone I0326 23:12:49.055576 3952 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-27 23:07:48 +0000 UTC" deadline="2026-03-27 19:43:28.641177433 +0000 UTC" I0326 23:12:49.055619 3952 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h30m39.585568341s" I0326 23:12:49.108708 3952 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:12:49.113903 3952 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:12:49.119076 3952 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:12:49.119097 3952 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:12:49.119107 3952 factory.go:636] Watch Factory start up complete, took: 652.811856ms I0326 23:12:49.119207 3952 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0326 23:12:49.156979 3952 default_node_network_controller.go:709] Initializing the default node network controller I0326 23:12:49.232382 3952 udn_isolation.go:101] Starting UDN host isolation manager I0326 23:12:49.242409 3952 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0326 23:12:49.332717 3952 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0326 23:12:49.332849 3952 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0326 23:12:49.332870 3952 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0326 23:12:49.367334 3952 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0326 23:12:49.367399 3952 default_node_network_controller.go:808] Node ip-10-0-0-90.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0326 23:12:49.373228 3952 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.90"] k8s.ovn.org/zone-name:ip-10-0-0-90.ec2.internal] on node ip-10-0-0-90.ec2.internal I0326 23:12:49.395495 3952 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0326 23:12:49.400997 3952 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0326 23:12:49.401238 3952 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0326 23:12:49.401475 3952 gateway_init.go:263] Preparing Gateway I0326 23:12:49.401487 3952 gateway_shared_intf.go:1658] Creating new gateway I0326 23:12:49.521417 3952 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:12:49.528114 3952 controller_manager.go:427] Waiting for node in zone sync took: 500.587447ms I0326 23:12:49.528459 3952 iptables.go:108] Creating table: filter chain: OUTPUT I0326 23:12:49.533264 3952 controller_manager.go:608] Switching to transit router for layer2 networks I0326 23:12:49.533283 3952 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-90.ec2.internal I0326 23:12:49.535537 3952 gateway_shared_intf.go:1787] Gateway Creation Complete I0326 23:12:49.535555 3952 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-90.ec2.internal","mac-address":"12:96:e0:eb:35:39","ip-addresses":["10.0.0.90/24"],"ip-address":"10.0.0.90/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:adfdde01-cb0d-4f45-8aab-392f0df67254 k8s.ovn.org/node-encap-ips:["10.0.0.90"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.90/24"} k8s.ovn.org/zone-name:ip-10-0-0-90.ec2.internal] on node ip-10-0-0-90.ec2.internal I0326 23:12:49.553711 3952 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:12:49.560009 3952 controller_manager.go:288] SCTP support detected in OVN I0326 23:12:49.560637 3952 services_controller.go:69] Creating services controller for network=default I0326 23:12:49.560737 3952 udn_enabled_svc.go:80] Waiting for service informer to sync I0326 23:12:49.560883 3952 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0326 23:12:49.560916 3952 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0326 23:12:49.560934 3952 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0326 23:12:49.560945 3952 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:12:49.561107 3952 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0326 23:12:49.561156 3952 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0326 23:12:49.561188 3952 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 24.697µs I0326 23:12:49.561200 3952 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 43.56µs I0326 23:12:49.561208 3952 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0326 23:12:49.561208 3952 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 61.68µs I0326 23:12:49.561221 3952 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0326 23:12:49.561232 3952 default_network_controller.go:321] Starting the default network controller I0326 23:12:49.561246 3952 udn_enabled_svc.go:95] Performing full resync I0326 23:12:49.561309 3952 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 15.25µs I0326 23:12:49.561518 3952 udn_enabled_svc.go:99] Waiting for handler to sync I0326 23:12:49.561531 3952 udn_enabled_svc.go:104] Starting worker I0326 23:12:49.570061 3952 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0326 23:12:49.570125 3952 repair.go:33] Syncing exgw routes took 49.198µs I0326 23:12:49.570145 3952 default_network_controller.go:407] Starting all the Watchers... I0326 23:12:49.570341 3952 namespace.go:98] [openshift-host-network] adding namespace I0326 23:12:49.570356 3952 namespace.go:98] [openshift-apiserver] adding namespace I0326 23:12:49.570344 3952 namespace.go:98] [openshift-route-controller-manager] adding namespace I0326 23:12:49.570362 3952 namespace.go:98] [openshift-network-operator] adding namespace I0326 23:12:49.570366 3952 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0326 23:12:49.570342 3952 namespace.go:98] [openshift-config] adding namespace I0326 23:12:49.570341 3952 namespace.go:98] [openshift-multus] adding namespace I0326 23:12:49.570385 3952 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0326 23:12:49.570391 3952 namespace.go:98] [openshift-network-console] adding namespace I0326 23:12:49.570398 3952 namespace.go:98] [openshift-backplane-tam] adding namespace I0326 23:12:49.570403 3952 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0326 23:12:49.570406 3952 namespace.go:98] [openshift-backplane-cee] adding namespace I0326 23:12:49.570415 3952 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0326 23:12:49.570436 3952 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0326 23:12:49.570341 3952 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0326 23:12:49.571056 3952 namespace.go:102] [openshift-host-network] adding namespace took 698.28µs I0326 23:12:49.571072 3952 namespace.go:98] [openshift-backplane-csm] adding namespace I0326 23:12:49.571606 3952 namespace.go:102] [openshift-backplane-csm] adding namespace took 526.813µs I0326 23:12:49.571627 3952 namespace.go:98] [openshift-authentication] adding namespace I0326 23:12:49.572305 3952 namespace.go:102] [openshift-authentication] adding namespace took 670.778µs I0326 23:12:49.572323 3952 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0326 23:12:49.572958 3952 namespace.go:102] [openshift-apiserver] adding namespace took 2.592037ms I0326 23:12:49.572975 3952 namespace.go:98] [openshift-authentication-operator] adding namespace I0326 23:12:49.573709 3952 default_node_network_controller.go:518] Node connection status = connected I0326 23:12:49.573764 3952 namespace.go:102] [openshift-network-operator] adding namespace took 3.388555ms I0326 23:12:49.573801 3952 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0326 23:12:49.574461 3952 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 4.082153ms I0326 23:12:49.574476 3952 namespace.go:98] [openshift-backplane-mobb] adding namespace I0326 23:12:49.575111 3952 namespace.go:102] [openshift-route-controller-manager] adding namespace took 4.74047ms I0326 23:12:49.575126 3952 namespace.go:98] [kube-system] adding namespace W0326 23:12:49.575697 3952 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-snttp: pod openshift-multus/network-metrics-daemon-snttp: no pod IPs found I0326 23:12:49.575738 3952 namespace.go:102] [openshift-config] adding namespace took 5.354294ms I0326 23:12:49.575752 3952 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0326 23:12:49.576402 3952 namespace.go:102] [openshift-multus] adding namespace took 6.013509ms I0326 23:12:49.576423 3952 namespace.go:98] [default] adding namespace I0326 23:12:49.577099 3952 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.707102ms I0326 23:12:49.577113 3952 namespace.go:98] [openshift-package-operator] adding namespace I0326 23:12:49.577854 3952 namespace.go:102] [openshift-network-console] adding namespace took 7.457732ms I0326 23:12:49.577864 3952 namespace.go:98] [openshift-backplane-cse] adding namespace I0326 23:12:49.578526 3952 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.11874ms I0326 23:12:49.578544 3952 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0326 23:12:49.579188 3952 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.772824ms I0326 23:12:49.579202 3952 namespace.go:98] [openshift-network-node-identity] adding namespace I0326 23:12:49.579920 3952 namespace.go:102] [openshift-backplane-cee] adding namespace took 9.499285ms I0326 23:12:49.579944 3952 namespace.go:98] [openshift-kube-apiserver] adding namespace I0326 23:12:49.580551 3952 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 10.125437ms I0326 23:12:49.580568 3952 namespace.go:98] [openshift-operators-redhat] adding namespace I0326 23:12:49.581094 3952 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 10.646258ms I0326 23:12:49.581113 3952 namespace.go:98] [openshift-ingress] adding namespace I0326 23:12:49.581517 3952 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 11.040514ms I0326 23:12:49.581531 3952 namespace.go:98] [openshift-kube-scheduler] adding namespace I0326 23:12:49.581898 3952 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 9.56781ms I0326 23:12:49.581914 3952 namespace.go:98] [openshift-ingress-operator] adding namespace I0326 23:12:49.582451 3952 namespace.go:102] [openshift-authentication-operator] adding namespace took 9.471313ms I0326 23:12:49.582460 3952 namespace.go:98] [openshift-etcd] adding namespace I0326 23:12:49.582974 3952 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 9.168272ms I0326 23:12:49.582983 3952 namespace.go:98] [openshift-apiserver-operator] adding namespace I0326 23:12:49.583811 3952 namespace.go:102] [openshift-backplane-mobb] adding namespace took 9.327849ms I0326 23:12:49.583830 3952 namespace.go:98] [kube-public] adding namespace I0326 23:12:49.584402 3952 namespace.go:102] [kube-system] adding namespace took 9.269539ms I0326 23:12:49.584417 3952 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0326 23:12:49.584957 3952 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 9.199926ms I0326 23:12:49.584970 3952 namespace.go:98] [openshift-machine-config-operator] adding namespace I0326 23:12:49.585495 3952 namespace.go:102] [default] adding namespace took 9.067417ms I0326 23:12:49.585506 3952 namespace.go:98] [openshift-marketplace] adding namespace I0326 23:12:49.585987 3952 namespace.go:102] [openshift-package-operator] adding namespace took 8.865831ms I0326 23:12:49.586006 3952 namespace.go:98] [kube-node-lease] adding namespace I0326 23:12:49.586462 3952 namespace.go:102] [openshift-backplane-cse] adding namespace took 8.592356ms I0326 23:12:49.586480 3952 namespace.go:98] [openshift-service-ca] adding namespace I0326 23:12:49.586932 3952 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 8.38077ms I0326 23:12:49.586951 3952 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0326 23:12:49.587479 3952 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.271778ms I0326 23:12:49.587496 3952 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0326 23:12:49.587998 3952 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.045754ms I0326 23:12:49.588017 3952 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0326 23:12:49.588588 3952 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.013196ms I0326 23:12:49.588603 3952 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0326 23:12:49.589116 3952 namespace.go:102] [openshift-ingress] adding namespace took 7.996339ms I0326 23:12:49.589132 3952 namespace.go:98] [openshift-insights] adding namespace I0326 23:12:49.589494 3952 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.957608ms I0326 23:12:49.589507 3952 namespace.go:98] [openshift-dns] adding namespace I0326 23:12:49.589968 3952 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.047471ms I0326 23:12:49.589982 3952 namespace.go:98] [openshift-console-user-settings] adding namespace I0326 23:12:49.590380 3952 namespace.go:102] [openshift-etcd] adding namespace took 7.915491ms I0326 23:12:49.590393 3952 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0326 23:12:49.590958 3952 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.969661ms I0326 23:12:49.590971 3952 namespace.go:98] [openshift-backplane] adding namespace I0326 23:12:49.591512 3952 namespace.go:102] [kube-public] adding namespace took 7.675243ms I0326 23:12:49.591525 3952 namespace.go:98] [openshift-console-operator] adding namespace I0326 23:12:49.592008 3952 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.584797ms I0326 23:12:49.592025 3952 namespace.go:98] [openshift-network-diagnostics] adding namespace I0326 23:12:49.592538 3952 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.561208ms I0326 23:12:49.592558 3952 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0326 23:12:49.593094 3952 namespace.go:102] [openshift-marketplace] adding namespace took 7.582615ms I0326 23:12:49.593107 3952 namespace.go:98] [openshift-dns-operator] adding namespace I0326 23:12:49.593521 3952 namespace.go:102] [kube-node-lease] adding namespace took 7.507016ms I0326 23:12:49.593538 3952 namespace.go:98] [openshift-config-managed] adding namespace I0326 23:12:49.594004 3952 namespace.go:102] [openshift-service-ca] adding namespace took 7.518844ms I0326 23:12:49.594015 3952 namespace.go:98] [openshift] adding namespace I0326 23:12:49.594472 3952 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.516464ms I0326 23:12:49.594482 3952 namespace.go:98] [open-cluster-management-2p9mn5g6ejev430bbu2pa8d8q491egfs] adding namespace I0326 23:12:49.594942 3952 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.44055ms I0326 23:12:49.594960 3952 namespace.go:98] [openshift-customer-monitoring] adding namespace I0326 23:12:49.595432 3952 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.409928ms I0326 23:12:49.595446 3952 namespace.go:98] [openshift-monitoring] adding namespace I0326 23:12:49.596007 3952 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.398898ms I0326 23:12:49.596020 3952 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0326 23:12:49.596585 3952 namespace.go:102] [openshift-insights] adding namespace took 7.449286ms I0326 23:12:49.596596 3952 namespace.go:98] [openshift-config-operator] adding namespace I0326 23:12:49.597132 3952 namespace.go:102] [openshift-dns] adding namespace took 7.618671ms I0326 23:12:49.597146 3952 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0326 23:12:49.597666 3952 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.67765ms I0326 23:12:49.598168 3952 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.769946ms I0326 23:12:49.598184 3952 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0326 23:12:49.598674 3952 namespace.go:102] [openshift-backplane] adding namespace took 7.696241ms I0326 23:12:49.598692 3952 namespace.go:98] [openshift-service-ca-operator] adding namespace W0326 23:12:49.599199 3952 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-wmdfm: pod openshift-network-diagnostics/network-check-target-wmdfm: no pod IPs found I0326 23:12:49.599250 3952 namespace.go:102] [openshift-console-operator] adding namespace took 7.717213ms I0326 23:12:49.599268 3952 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0326 23:12:49.599807 3952 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.766001ms I0326 23:12:49.599822 3952 namespace.go:98] [openshift-must-gather-operator] adding namespace I0326 23:12:49.600300 3952 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.731377ms I0326 23:12:49.600313 3952 namespace.go:98] [openshift-console] adding namespace I0326 23:12:49.600680 3952 namespace.go:102] [openshift-dns-operator] adding namespace took 7.566426ms I0326 23:12:49.600694 3952 namespace.go:98] [openshift-operators] adding namespace I0326 23:12:49.601294 3952 namespace.go:102] [openshift-config-managed] adding namespace took 7.750293ms I0326 23:12:49.601312 3952 namespace.go:98] [openshift-cluster-version] adding namespace I0326 23:12:49.601797 3952 namespace.go:102] [openshift] adding namespace took 7.77657ms I0326 23:12:49.601814 3952 namespace.go:98] [openshift-infra] adding namespace I0326 23:12:49.602390 3952 namespace.go:102] [open-cluster-management-2p9mn5g6ejev430bbu2pa8d8q491egfs] adding namespace took 7.902179ms I0326 23:12:49.602407 3952 namespace.go:98] [openshift-machine-api] adding namespace I0326 23:12:49.602907 3952 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.940084ms I0326 23:12:49.602921 3952 namespace.go:98] [dedicated-admin] adding namespace I0326 23:12:49.603943 3952 namespace.go:102] [openshift-monitoring] adding namespace took 8.490605ms I0326 23:12:49.603963 3952 namespace.go:98] [openshift-image-registry] adding namespace I0326 23:12:49.604574 3952 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 8.547561ms I0326 23:12:49.604591 3952 namespace.go:98] [openshift-backplane-srep] adding namespace I0326 23:12:49.605141 3952 namespace.go:102] [openshift-config-operator] adding namespace took 8.53813ms I0326 23:12:49.605580 3952 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 8.426459ms I0326 23:12:49.605595 3952 namespace.go:98] [openshift-controller-manager] adding namespace I0326 23:12:49.606095 3952 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.904105ms I0326 23:12:49.606487 3952 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.787876ms I0326 23:12:49.607166 3952 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.885621ms I0326 23:12:49.607724 3952 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.894417ms I0326 23:12:49.607750 3952 namespace.go:98] [openshift-logging] adding namespace I0326 23:12:49.608379 3952 namespace.go:102] [openshift-console] adding namespace took 8.057509ms I0326 23:12:49.608905 3952 namespace.go:102] [openshift-operators] adding namespace took 8.204662ms I0326 23:12:49.608921 3952 namespace.go:98] [openshift-ingress-canary] adding namespace I0326 23:12:49.609445 3952 namespace.go:102] [openshift-cluster-version] adding namespace took 8.12545ms I0326 23:12:49.610037 3952 namespace.go:102] [openshift-infra] adding namespace took 8.216743ms I0326 23:12:49.610049 3952 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0326 23:12:49.610674 3952 namespace.go:102] [openshift-machine-api] adding namespace took 8.259091ms I0326 23:12:49.610692 3952 namespace.go:98] [openshift-node] adding namespace I0326 23:12:49.611318 3952 namespace.go:102] [dedicated-admin] adding namespace took 8.390669ms I0326 23:12:49.611962 3952 namespace.go:102] [openshift-image-registry] adding namespace took 7.991901ms I0326 23:12:49.611978 3952 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0326 23:12:49.612410 3952 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.813354ms I0326 23:12:49.612875 3952 namespace.go:102] [openshift-controller-manager] adding namespace took 7.273986ms I0326 23:12:49.612889 3952 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0326 23:12:49.613316 3952 namespace.go:102] [openshift-logging] adding namespace took 5.560165ms I0326 23:12:49.613754 3952 namespace.go:102] [openshift-ingress-canary] adding namespace took 4.826474ms I0326 23:12:49.614298 3952 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 4.243912ms I0326 23:12:49.614750 3952 namespace.go:102] [openshift-node] adding namespace took 4.052616ms I0326 23:12:49.615357 3952 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 3.37089ms I0326 23:12:49.615824 3952 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 2.928886ms I0326 23:12:49.616237 3952 zone_ic_handler.go:186] Time taken to create transit switch: 311.921µs I0326 23:12:49.616329 3952 master.go:532] Adding or Updating local node "ip-10-0-0-90.ec2.internal" for network "default" I0326 23:12:49.616866 3952 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-8.ec2.internal for the network default I0326 23:12:49.617061 3952 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-97.ec2.internal for the network default I0326 23:12:49.618092 3952 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-8.ec2.internal" on network "default" took: 1.23811ms I0326 23:12:49.618357 3952 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-97.ec2.internal" on network "default" took: 1.310292ms W0326 23:12:49.626636 3952 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-wmdfm: pod openshift-network-diagnostics/network-check-target-wmdfm: no pod IPs found W0326 23:12:49.626663 3952 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-snttp: pod openshift-multus/network-metrics-daemon-snttp: no pod IPs found I0326 23:12:49.634415 3952 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-90.ec2.internal for the network default I0326 23:12:49.635513 3952 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0326 23:12:49.635555 3952 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0326 23:12:49.635578 3952 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0326 23:12:49.635599 3952 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-8.ec2.internal in network "default" I0326 23:12:49.635666 3952 node_tracker.go:169] Node ip-10-0-1-8.ec2.internal switch + router changed, syncing services in network "default" I0326 23:12:49.635682 3952 services_controller.go:574] Full service sync requested for network=default I0326 23:12:49.635710 3952 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-97.ec2.internal in network "default" I0326 23:12:49.635748 3952 node_tracker.go:169] Node ip-10-0-2-97.ec2.internal switch + router changed, syncing services in network "default" I0326 23:12:49.635759 3952 services_controller.go:574] Full service sync requested for network=default I0326 23:12:49.635766 3952 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-90.ec2.internal in network "default" I0326 23:12:49.635828 3952 node_tracker.go:169] Node ip-10-0-0-90.ec2.internal switch + router changed, syncing services in network "default" I0326 23:12:49.635838 3952 services_controller.go:574] Full service sync requested for network=default I0326 23:12:49.736626 3952 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0326 23:12:49.736646 3952 services_controller.go:208] Setting up event handlers for services for network=default I0326 23:12:49.736741 3952 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0326 23:12:49.736802 3952 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0326 23:12:49.736818 3952 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0326 23:12:49.837427 3952 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0326 23:12:49.837455 3952 repair.go:60] Starting repairing loop for services I0326 23:12:49.837549 3952 repair.go:131] Deleted 0 stale service LBs I0326 23:12:49.837573 3952 repair.go:137] Deleted 0 stale Chassis Template Vars I0326 23:12:49.837590 3952 repair.go:62] Finished repairing loop for services: 137.975µs I0326 23:12:49.837616 3952 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0326 23:12:49.837626 3952 services_controller.go:254] Starting workers for network=default W0326 23:12:49.837864 3952 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0326 23:12:49.837886 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0326 23:12:49.837893 3952 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0326 23:12:49.837950 3952 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-snttp] creating logical port openshift-multus_network-metrics-daemon-snttp for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:49.838037 3952 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-r4q77 of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0326 23:12:49.838079 3952 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-snttp I0326 23:12:49.838210 3952 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-wmdfm] creating logical port openshift-network-diagnostics_network-check-target-wmdfm for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:49.838303 3952 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-wmdfm W0326 23:12:49.838680 3952 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:12:49.840830 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:12:49.840844 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:12:49.840850 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:12:49.843120 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:12:49.843136 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0326 23:12:49.853478 3952 pod.go:62] [openshift-multus/network-metrics-daemon-snttp] pod update took 15.448145ms I0326 23:12:49.853495 3952 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-snttp] addLogicalPort annotation time took 15.472825ms I0326 23:12:49.854411 3952 pods.go:271] [openshift-multus/network-metrics-daemon-snttp] addLogicalPort took 16.467693ms, libovsdb time 699.705µs I0326 23:12:49.855325 3952 pod.go:62] [openshift-network-diagnostics/network-check-target-wmdfm] pod update took 17.064367ms I0326 23:12:49.855348 3952 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-wmdfm] addLogicalPort annotation time took 17.088207ms I0326 23:12:49.856359 3952 pods.go:271] [openshift-network-diagnostics/network-check-target-wmdfm] addLogicalPort took 18.160316ms, libovsdb time 772.582µs I0326 23:12:49.857035 3952 admin_network_policy_controller.go:227] Starting controller default-network-controller I0326 23:12:49.857066 3952 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0326 23:12:49.857083 3952 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0326 23:12:49.857090 3952 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0326 23:12:49.857378 3952 repair.go:29] Repairing admin network policies took 280.874µs I0326 23:12:49.857546 3952 repair.go:92] Repairing baseline admin network policies took 157.865µs I0326 23:12:49.857898 3952 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0326 23:12:49.857929 3952 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0326 23:12:49.861093 3952 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0326 23:12:49.861121 3952 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0326 23:12:49.861136 3952 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0326 23:12:49.862574 3952 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0326 23:12:49.862600 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0326 23:12:49.862611 3952 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0326 23:12:49.864458 3952 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0326 23:12:49.864484 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0326 23:12:49.864495 3952 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0326 23:12:49.865225 3952 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0326 23:12:49.865246 3952 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0326 23:12:49.865256 3952 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0326 23:12:49.868270 3952 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0326 23:12:49.868290 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0326 23:12:49.868299 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0326 23:12:49.870257 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0326 23:12:49.870275 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0326 23:12:49.870283 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0326 23:12:49.870910 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:12:49.870933 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0326 23:12:49.870946 3952 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0326 23:12:49.871736 3952 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0326 23:12:49.871756 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0326 23:12:49.871788 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0326 23:12:49.873793 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:12:49.873811 3952 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0326 23:12:49.873821 3952 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0326 23:12:49.876488 3952 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0326 23:12:49.876512 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0326 23:12:49.876526 3952 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0326 23:12:49.877170 3952 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0326 23:12:49.877193 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0326 23:12:49.877203 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0326 23:12:49.877837 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0326 23:12:49.877860 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0326 23:12:49.877869 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0326 23:12:49.878260 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0326 23:12:49.878279 3952 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0326 23:12:49.878288 3952 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0326 23:12:49.878878 3952 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0326 23:12:49.878898 3952 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0326 23:12:49.878906 3952 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0326 23:12:49.879216 3952 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0326 23:12:49.879234 3952 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0326 23:12:49.879244 3952 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0326 23:12:49.880381 3952 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0326 23:12:49.880406 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0326 23:12:49.880414 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0326 23:12:49.880971 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0326 23:12:49.880993 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0326 23:12:49.881000 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0326 23:12:49.881479 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0326 23:12:49.881499 3952 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0326 23:12:49.881507 3952 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0326 23:12:49.882009 3952 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0326 23:12:49.882027 3952 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0326 23:12:49.882034 3952 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0326 23:12:49.882386 3952 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0326 23:12:49.882409 3952 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0326 23:12:49.882423 3952 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0326 23:12:49.884353 3952 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0326 23:12:49.884374 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0326 23:12:49.884382 3952 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0326 23:12:49.884727 3952 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0326 23:12:49.884749 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0326 23:12:49.885575 3952 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0326 23:12:49.885587 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:12:49.886346 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0326 23:12:49.886367 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0326 23:12:49.886394 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:12:49.887138 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0326 23:12:49.887157 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0326 23:12:49.887163 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0326 23:12:49.888507 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0326 23:12:49.888526 3952 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0326 23:12:49.888536 3952 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0326 23:12:49.890155 3952 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0326 23:12:49.890175 3952 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0326 23:12:49.890183 3952 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0326 23:12:49.890891 3952 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0326 23:12:49.890915 3952 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0326 23:12:49.890927 3952 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0326 23:12:49.892081 3952 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0326 23:12:49.892105 3952 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0326 23:12:49.892127 3952 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0326 23:12:49.893133 3952 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0326 23:12:49.893420 3952 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-r4q77 of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0326 23:12:49.899568 3952 egressfirewall.go:319] Starting EgressFirewall controller I0326 23:12:49.899593 3952 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0326 23:12:49.899618 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0326 23:12:49.899621 3952 controller.go:133] Adding controller egress-firewall-controller event handlers I0326 23:12:49.899632 3952 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0326 23:12:49.899653 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0326 23:12:49.899667 3952 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0326 23:12:49.899991 3952 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0326 23:12:49.900007 3952 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0326 23:12:49.900017 3952 egressqos.go:193] Setting up event handlers for EgressQoS I0326 23:12:49.900100 3952 egressfirewall.go:628] Syncing node "ip-10-0-0-90.ec2.internal" for egress firewall I0326 23:12:49.900114 3952 egressfirewall.go:628] Syncing node "ip-10-0-1-8.ec2.internal" for egress firewall I0326 23:12:49.900120 3952 egressfirewall.go:628] Syncing node "ip-10-0-2-97.ec2.internal" for egress firewall I0326 23:12:49.900174 3952 egressqos.go:245] Starting EgressQoS Controller I0326 23:12:49.900188 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0326 23:12:49.900199 3952 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0326 23:12:49.900209 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0326 23:12:49.900216 3952 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0326 23:12:49.900224 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0326 23:12:49.900230 3952 shared_informer.go:356] "Caches are synced" controller="egressqos" I0326 23:12:49.900234 3952 egressqos.go:259] Repairing EgressQoSes I0326 23:12:49.900238 3952 egressqos.go:399] Starting repairing loop for egressqos I0326 23:12:49.900311 3952 egressqos.go:401] Finished repairing loop for egressqos: 73.531µs I0326 23:12:49.900329 3952 egressservice_zone.go:132] Setting up event handlers for Egress Services I0326 23:12:49.900542 3952 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-90.ec2.internal I0326 23:12:49.900604 3952 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-90.ec2.internal : 62.778µs I0326 23:12:49.900885 3952 egressservice_zone.go:210] Starting Egress Services Controller I0326 23:12:49.900900 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:12:49.900911 3952 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:12:49.900921 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:12:49.900929 3952 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:12:49.900938 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:12:49.900945 3952 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:12:49.900950 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0326 23:12:49.900955 3952 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0326 23:12:49.900958 3952 egressservice_zone.go:228] Repairing Egress Services I0326 23:12:49.901265 3952 master_controller.go:88] Starting Admin Policy Based Route Controller I0326 23:12:49.901274 3952 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:12:49.901298 3952 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-90.ec2.internal I0326 23:12:49.901313 3952 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-90.ec2.internal: 15.215µs I0326 23:12:49.901324 3952 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-8.ec2.internal I0326 23:12:49.901334 3952 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-8.ec2.internal: 11.752µs I0326 23:12:49.901339 3952 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-97.ec2.internal I0326 23:12:49.901344 3952 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-97.ec2.internal: 5.595µs I0326 23:12:49.901572 3952 default_network_controller.go:568] Completing all the Watchers took 331.419189ms I0326 23:12:49.901587 3952 default_network_controller.go:572] Starting unidling controllers I0326 23:12:49.901604 3952 unidle.go:46] Registering OVN SB ControllerEvent handler I0326 23:12:49.901609 3952 unidle.go:63] Populating Initial ContollerEvent events I0326 23:12:49.901619 3952 unidle.go:79] Setting up event handlers for services I0326 23:12:49.901663 3952 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:12:49.907345 3952 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:12:50.073917 3952 default_node_network_controller.go:518] Node connection status = connected I0326 23:12:50.091922 3952 bridgeconfig.go:537] Gateway is ready I0326 23:12:50.091947 3952 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0326 23:12:50.102787 3952 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.90. Updating OVN encap IP. I0326 23:12:50.107537 3952 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.90 - it is already configured I0326 23:12:50.107551 3952 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.90/24:{} 172.20.0.1/32:{}]. Updating annotations. I0326 23:12:50.107985 3952 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.90/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-90.ec2.internal","mac-address":"12:96:e0:eb:35:39","ip-addresses":["10.0.0.90/24"],"ip-address":"10.0.0.90/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:adfdde01-cb0d-4f45-8aab-392f0df67254 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.90/24"}] on node ip-10-0-0-90.ec2.internal I0326 23:12:50.125979 3952 egressfirewall.go:628] Syncing node "ip-10-0-0-90.ec2.internal" for egress firewall I0326 23:12:50.126032 3952 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-90.ec2.internal I0326 23:12:50.126044 3952 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-90.ec2.internal: 15.039µs I0326 23:12:50.126069 3952 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-90.ec2.internal in network "default" I0326 23:12:50.126086 3952 master.go:532] Adding or Updating local node "ip-10-0-0-90.ec2.internal" for network "default" I0326 23:12:50.126143 3952 node_tracker.go:169] Node ip-10-0-0-90.ec2.internal switch + router changed, syncing services in network "default" I0326 23:12:50.126161 3952 services_controller.go:574] Full service sync requested for network=default W0326 23:12:50.126351 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0326 23:12:50.126365 3952 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:12:50.126370 3952 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0326 23:12:50.126540 3952 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0326 23:12:50.126570 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:12:50.126605 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:12:50.126621 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0326 23:12:50.126798 3952 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.90 W0326 23:12:50.126807 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:12:50.126824 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0326 23:12:50.126928 3952 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-90.ec2.internal I0326 23:12:50.131392 3952 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-90.ec2.internal. Updating no re-route policies I0326 23:12:50.131463 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-wmdfm I0326 23:12:50.131464 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-2nk58 I0326 23:12:50.131478 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-wmdfm I0326 23:12:50.131478 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dl9l2 I0326 23:12:50.131482 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-2nk58 I0326 23:12:50.131493 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dl9l2 I0326 23:12:50.131503 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-2nk58 after 0 failed attempt(s) I0326 23:12:50.131512 3952 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-wmdfm] creating logical port openshift-network-diagnostics_network-check-target-wmdfm for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:50.131516 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fvqhj I0326 23:12:50.131517 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dl9l2 after 0 failed attempt(s) I0326 23:12:50.131518 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-knbch I0326 23:12:50.131534 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-knbch I0326 23:12:50.131538 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-zjr2b I0326 23:12:50.131535 3952 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-90.ec2.internal I0326 23:12:50.131542 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-v4zbq I0326 23:12:50.131551 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-nfw2s I0326 23:12:50.131552 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-zjr2b I0326 23:12:50.131559 3952 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-90.ec2.internal I0326 23:12:50.131564 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-nfw2s I0326 23:12:50.131564 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fvqhj I0326 23:12:50.131564 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-v4zbq I0326 23:12:50.131510 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-7hwkm I0326 23:12:50.131574 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-zjr2b after 0 failed attempt(s) I0326 23:12:50.131579 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-7hwkm I0326 23:12:50.131579 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-snttp I0326 23:12:50.131582 3952 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-90.ec2.internal after 0 failed attempt(s) I0326 23:12:50.131588 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-fvqhj after 0 failed attempt(s) I0326 23:12:50.131584 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-nfw2s after 0 failed attempt(s) I0326 23:12:50.131593 3952 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-90.ec2.internal I0326 23:12:50.131566 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-knbch after 0 failed attempt(s) I0326 23:12:50.131597 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-v4zbq after 0 failed attempt(s) I0326 23:12:50.131603 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-90.ec2.internal I0326 23:12:50.131595 3952 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-snttp I0326 23:12:50.131625 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-90.ec2.internal after 0 failed attempt(s) I0326 23:12:50.131624 3952 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-v4q6k I0326 23:12:50.131652 3952 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-snttp] creating logical port openshift-multus_network-metrics-daemon-snttp for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:50.131666 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-7hwkm after 0 failed attempt(s) I0326 23:12:50.131677 3952 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-v4q6k I0326 23:12:50.131699 3952 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-v4q6k after 0 failed attempt(s) I0326 23:12:50.132421 3952 pods.go:271] [openshift-network-diagnostics/network-check-target-wmdfm] addLogicalPort took 919.454µs, libovsdb time 251.948µs I0326 23:12:50.132434 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-wmdfm after 0 failed attempt(s) I0326 23:12:50.132581 3952 pods.go:271] [openshift-multus/network-metrics-daemon-snttp] addLogicalPort took 941.272µs, libovsdb time 336.303µs I0326 23:12:50.132594 3952 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-snttp after 0 failed attempt(s) I0326 23:12:50.147388 3952 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0326 23:12:50.160085 3952 iptables.go:108] Creating table: mangle chain: OUTPUT I0326 23:12:50.163910 3952 iptables.go:108] Creating table: nat chain: OUTPUT I0326 23:12:50.194126 3952 iptables.go:108] Creating table: nat chain: PREROUTING I0326 23:12:50.278791 3952 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:12:50.297813 3952 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0326 23:12:50.297962 3952 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0326 23:12:50.298020 3952 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0326 23:12:50.298125 3952 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0326 23:12:50.298193 3952 gateway_init.go:325] Gateway and management port readiness took 728.186206ms I0326 23:12:50.298246 3952 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0326 23:12:50.298286 3952 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0326 23:12:50.298305 3952 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0326 23:12:50.298318 3952 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:12:50.298410 3952 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0326 23:12:50.298558 3952 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 52.61µs I0326 23:12:50.298833 3952 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0326 23:12:50.298869 3952 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 23.069µs I0326 23:12:50.298881 3952 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 47.411µs I0326 23:12:50.298889 3952 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0326 23:12:50.298903 3952 nad_controller.go:173] [node-nad-controller NAD controller]: started I0326 23:12:50.298912 3952 default_node_network_controller.go:898] Starting the default node network controller I0326 23:12:50.298919 3952 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0326 23:12:50.298992 3952 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.157µs I0326 23:12:50.299091 3952 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0326 23:12:50.299102 3952 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0326 23:12:50.299130 3952 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a91210 0xc0002ec150 0xc0002ec1c0 0xc0002ec230 0xc0002ec2a0 0xc0002ec380 0xc0002ec3f0 0xc000aa9720 0xc0002ec460 0xc0002ec4d0 0xc000a02280 0xc006592fc0 0xc0002ec540 map[0x2f67040:0xc002884a20 0x2f67580:0xc002884d20 0x2f68000:0xc00299ecc0 0x2f682a0:0xc0028849c0 0x2f687e0:0xc002884a80 0x2f68a80:0xc002884ae0 0x2f68d20:0xc002884b40 0x2f69500:0xc002884d80 0x2f697a0:0xc002884de0 0x2f93d40:0xc000d26cc0 0x2f940e0:0xc000d26d20 0x2f94bc0:0xc000a417a0 0x2f96520:0xc001b0c000 0x2f97000:0xc00221be00 0x2f97e80:0xc001b0c060] 0xc000aaa620 0} I0326 23:12:50.299188 3952 gateway.go:151] Starting gateway service sync I0326 23:12:50.299718 3952 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0326 23:12:50.302375 3952 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0326 23:12:50.315909 3952 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0326 23:12:50.317243 3952 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0326 23:12:50.318504 3952 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0326 23:12:50.353900 3952 gateway.go:168] Gateway service sync done. Time taken: 54.697705ms I0326 23:12:50.354584 3952 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31200 I0326 23:12:50.354892 3952 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a91210 0xc0002ec150 0xc0002ec1c0 0xc0002ec230 0xc0002ec2a0 0xc0002ec380 0xc0002ec3f0 0xc000aa9720 0xc0002ec460 0xc0002ec4d0 0xc000a02280 0xc006592fc0 0xc0002ec540 map[0x2f67040:0xc002884a20 0x2f67580:0xc002884d20 0x2f68000:0xc00299ecc0 0x2f682a0:0xc0028849c0 0x2f687e0:0xc002884a80 0x2f68a80:0xc002884ae0 0x2f68d20:0xc002884b40 0x2f69500:0xc002884d80 0x2f697a0:0xc002884de0 0x2f93d40:0xc000d26cc0 0x2f940e0:0xc000d26d20 0x2f94bc0:0xc000a417a0 0x2f96520:0xc001b0c000 0x2f97000:0xc00221be00 0x2f97e80:0xc001b0c060] 0xc000aaa620 0} I0326 23:12:50.355532 3952 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a91210 0xc0002ec150 0xc0002ec1c0 0xc0002ec230 0xc0002ec2a0 0xc0002ec380 0xc0002ec3f0 0xc000aa9720 0xc0002ec460 0xc0002ec4d0 0xc000a02280 0xc006592fc0 0xc0002ec540 map[0x2f67040:0xc002884a20 0x2f67580:0xc002884d20 0x2f68000:0xc00299ecc0 0x2f682a0:0xc0028849c0 0x2f687e0:0xc002884a80 0x2f68a80:0xc002884ae0 0x2f68d20:0xc002884b40 0x2f69500:0xc002884d80 0x2f697a0:0xc002884de0 0x2f93d40:0xc000d26cc0 0x2f940e0:0xc000d26d20 0x2f94bc0:0xc000a417a0 0x2f96520:0xc001b0c000 0x2f97000:0xc00221be00 0x2f97e80:0xc001b0c060] 0xc000aaa620 0} I0326 23:12:50.355585 3952 gateway.go:519] Reconciling gateway with updates I0326 23:12:50.361350 3952 gateway.go:326] Spawning Conntrack Rule Check Thread I0326 23:12:50.361443 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:12:50.361457 3952 default_node_network_controller.go:943] Gateway and management port readiness took 62.344072ms I0326 23:12:50.361469 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:12:50.361468 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:12:50.361469 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:12:50.361483 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:12:50.361489 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:12:50.361486 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:12:50.361505 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:12:50.361512 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:12:50.361524 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:12:50.361455 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:12:50.361538 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:12:50.361542 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361542 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:12:50.361549 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:12:50.361555 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:12:50.361560 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:12:50.361557 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:12:50.361561 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:12:50.361568 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:12:50.361530 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:12:50.361574 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:12:50.361568 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:12:50.361590 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:12:50.361591 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0326 23:12:50.361596 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:12:50.361592 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0326 23:12:50.361603 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:12:50.361609 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:12:50.361616 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:12:50.361619 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:12:50.361623 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:12:50.361628 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361630 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0326 23:12:50.361638 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/console I0326 23:12:50.361643 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:12:50.361646 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/console I0326 23:12:50.361649 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:12:50.361548 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:12:50.361669 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:12:50.361679 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/console after 0 failed attempt(s) I0326 23:12:50.361651 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361684 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:12:50.361690 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:12:50.361694 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:12:50.361548 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:12:50.361703 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:12:50.361712 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361714 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:12:50.361720 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:12:50.361725 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:12:50.361722 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:12:50.361547 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:12:50.361739 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:12:50.361592 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:12:50.361743 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:12:50.361746 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:12:50.361524 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:12:50.361759 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:12:50.361762 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:12:50.361765 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:12:50.361792 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:12:50.361739 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:12:50.361797 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:12:50.361804 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:12:50.361810 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:12:50.361815 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:12:50.361698 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:12:50.361813 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:12:50.361831 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:12:50.361831 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:12:50.361834 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:12:50.361836 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:12:50.361837 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:12:50.361843 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:12:50.361850 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:12:50.361849 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:12:50.361854 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:12:50.361856 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:12:50.361866 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:12:50.361875 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:12:50.361877 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:12:50.361617 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:12:50.361888 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:12:50.361916 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:12:50.361815 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361728 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361566 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:12:50.361942 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:12:50.361535 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:12:50.361945 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:12:50.361960 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:12:50.361572 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:12:50.362044 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:12:50.362051 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:12:50.362061 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:12:50.362067 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:12:50.362075 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:12:50.362076 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:12:50.362065 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:12:50.362084 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:12:50.362087 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:12:50.362049 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:12:50.362091 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/downloads I0326 23:12:50.362104 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:12:50.362124 3952 port_claim.go:71] Svc openshift-ingress/router-default: port 32402 is already open, no action needed I0326 23:12:50.362125 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:12:50.362138 3952 port_claim.go:71] Svc openshift-ingress/router-default: port 32577 is already open, no action needed I0326 23:12:50.362140 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:12:50.362128 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:12:50.362108 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/downloads I0326 23:12:50.362072 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:12:50.361572 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:12:50.362405 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:12:50.361599 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:12:50.362442 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:12:50.361604 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0326 23:12:50.362456 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0326 23:12:50.361686 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:12:50.361698 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook after 0 failed attempt(s) I0326 23:12:50.361702 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:12:50.361814 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:12:50.361820 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:12:50.361566 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:12:50.361569 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:12:50.361537 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:12:50.362536 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:12:50.361577 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:12:50.362265 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:12:50.363663 3952 node_ip_handler_linux.go:160] Node IP manager is running W0326 23:12:50.403536 3952 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:12:50.403792 3952 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:12:50.404021 3952 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0326 23:12:50.412382 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:12:50.412455 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:12:50.412467 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:12:50.412474 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:12:50.412479 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:12:50.412485 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) W0326 23:12:50.442616 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0326 23:12:50.452661 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:12:50.452696 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:12:50.452710 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:12:50.452719 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/downloads after 0 failed attempt(s) I0326 23:12:50.493237 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:12:50.493271 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:12:50.493307 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:12:50.493317 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:12:50.493327 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:12:50.493337 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:12:50.493346 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:12:52.158592 3952 default_node_network_controller.go:1364] Starting node controller node sync W0326 23:12:52.158645 3952 egressip_healthcheck.go:76] Health checking using insecure connection I0326 23:12:52.158682 3952 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0326 23:12:52.169818 3952 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.208964ms I0326 23:12:52.169929 3952 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-97.ec2.internal", IP: 10.0.2.97 to PMTUD blocking rules I0326 23:12:52.169932 3952 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-8.ec2.internal", IP: 10.0.1.8 to PMTUD blocking rules I0326 23:12:52.169940 3952 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-97.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:12:52.169944 3952 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-8.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:12:52.181235 3952 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0326 23:12:52.181496 3952 egressservice_node.go:93] Setting up event handlers for Egress Services I0326 23:12:52.181648 3952 egressservice_node.go:183] Starting Egress Services Controller I0326 23:12:52.181663 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:12:52.181675 3952 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:12:52.181680 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:12:52.181692 3952 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:12:52.181701 3952 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:12:52.181705 3952 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:12:52.181709 3952 egressservice_node.go:197] Repairing Egress Services I0326 23:12:52.191402 3952 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0326 23:12:52.191413 3952 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:12:52.191553 3952 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0326 23:12:52.191562 3952 link_network_manager.go:117] Link manager is running I0326 23:12:52.191644 3952 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0326 23:12:52.191713 3952 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0326 23:12:52.192417 3952 vrf_manager.go:128] VRF manager is running I0326 23:12:52.192582 3952 node_controller_manager.go:437] Removing flows to drop GARP I0326 23:12:52.192593 3952 gateway.go:519] Reconciling gateway with updates I0326 23:12:52.218629 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:12:52.218650 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:12:52.218650 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:12:52.218662 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:12:52.218667 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:12:52.218671 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:12:52.218673 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:12:52.218674 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/downloads I0326 23:12:52.218680 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:12:52.218684 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:12:52.218692 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:12:52.218692 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/downloads I0326 23:12:52.218700 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:12:52.218707 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:12:52.218708 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:12:52.218652 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:12:52.218716 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:12:52.218721 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:12:52.218726 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:12:52.218728 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/console I0326 23:12:52.218736 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:12:52.218636 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:12:52.218739 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:12:52.218739 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:12:52.218746 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/console I0326 23:12:52.218753 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:12:52.218751 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:12:52.218757 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:12:52.218758 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator I0326 23:12:52.218763 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:12:52.218766 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:12:52.218791 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator I0326 23:12:52.218792 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:12:52.218730 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:12:52.218799 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator after 0 failed attempt(s) I0326 23:12:52.218797 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:12:52.218805 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/downloads after 0 failed attempt(s) I0326 23:12:52.218802 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:12:52.218812 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:12:52.218811 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:12:52.218818 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:12:52.218820 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0326 23:12:52.218826 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:12:52.218827 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:12:52.218829 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:12:52.218830 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0326 23:12:52.218829 3952 port_claim.go:71] Svc openshift-ingress/router-default: port 32402 is already open, no action needed I0326 23:12:52.218837 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:12:52.218840 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:12:52.218838 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:12:52.218842 3952 port_claim.go:71] Svc openshift-ingress/router-default: port 32577 is already open, no action needed I0326 23:12:52.218847 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:12:52.218851 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:12:52.218858 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:12:52.218860 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:12:52.218864 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:12:52.218870 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:12:52.218875 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:12:52.218879 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:12:52.218886 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0326 23:12:52.218892 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook after 0 failed attempt(s) I0326 23:12:52.218897 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/console after 0 failed attempt(s) I0326 23:12:52.218900 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0326 23:12:52.218704 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:12:52.218909 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0326 23:12:52.218817 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:12:52.218920 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:12:52.218922 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:12:52.218919 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:12:52.218929 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:12:52.218932 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:12:52.218932 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:12:52.218940 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:12:52.218945 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:12:52.218953 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:12:52.218664 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:12:52.218993 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:12:52.218828 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:12:52.219006 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:12:52.219016 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:12:52.219023 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:12:52.219024 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:12:52.219031 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:12:52.218783 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:12:52.218791 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:12:52.218718 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:12:52.218741 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:12:52.219077 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:12:52.218818 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:12:52.219085 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:12:52.219084 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:12:52.219090 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:12:52.219077 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:12:52.219095 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:12:52.219096 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:12:52.219101 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:12:52.219105 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:12:52.219103 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:12:52.219109 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:12:52.219096 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:12:52.219161 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:12:52.218919 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:12:52.219170 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:12:52.218916 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:12:52.218925 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:12:52.218936 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:12:52.219296 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:12:52.219079 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:12:52.219086 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:12:52.219349 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:12:52.219088 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:12:52.218819 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:12:52.219430 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:12:52.218849 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:12:52.219463 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:12:52.219098 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:12:52.218908 3952 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:12:52.219527 3952 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:12:52.256247 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:12:52.256271 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:12:52.256284 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:12:52.256294 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:12:52.256302 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:12:52.293474 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:12:52.293505 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:12:52.293525 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:12:52.293535 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:12:52.293541 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:12:52.293547 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:12:52.293554 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:12:52.293561 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:12:52.339866 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:12:52.339892 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:12:52.339901 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:12:52.339908 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:12:52.339922 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:12:52.339940 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:12:52.339958 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:12:52.339984 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:12:52.339998 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:12:52.340008 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:12:52.340017 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:12:52.340027 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:12:52.340045 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:12:52.340055 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:12:52.340061 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:12:52.340082 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:12:52.340088 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:12:52.340094 3952 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:12:53.836908 3952 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-90.ec2.internal I0326 23:12:53.836938 3952 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-90.ec2.internal: 36.822µs I0326 23:12:53.869576 3952 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7fc9774446-hnkrj] creating logical port openshift-image-registry_image-registry-7fc9774446-hnkrj for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:53.869696 3952 kube.go:256] Updating pod openshift-image-registry/image-registry-7fc9774446-hnkrj I0326 23:12:53.887055 3952 pod.go:62] [openshift-image-registry/image-registry-7fc9774446-hnkrj] pod update took 17.393031ms I0326 23:12:53.887083 3952 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7fc9774446-hnkrj] addLogicalPort annotation time took 17.425234ms I0326 23:12:53.888383 3952 pods.go:271] [openshift-image-registry/image-registry-7fc9774446-hnkrj] addLogicalPort took 18.819583ms, libovsdb time 960.176µs I0326 23:12:53.930816 3952 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-kzq6d] creating logical port openshift-ingress-canary_ingress-canary-kzq6d for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:53.930904 3952 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-kzq6d I0326 23:12:53.948271 3952 pod.go:62] [openshift-ingress-canary/ingress-canary-kzq6d] pod update took 17.396138ms I0326 23:12:53.948297 3952 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-kzq6d] addLogicalPort annotation time took 17.425777ms I0326 23:12:53.949835 3952 pods.go:271] [openshift-ingress-canary/ingress-canary-kzq6d] addLogicalPort took 19.033863ms, libovsdb time 1.307855ms I0326 23:12:53.949882 3952 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-8hnkh] creating logical port openshift-insights_insights-runtime-extractor-8hnkh for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:53.949945 3952 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-8hnkh I0326 23:12:53.966470 3952 pod.go:62] [openshift-insights/insights-runtime-extractor-8hnkh] pod update took 16.543356ms I0326 23:12:53.966493 3952 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-8hnkh] addLogicalPort annotation time took 16.570997ms I0326 23:12:53.967415 3952 pods.go:271] [openshift-insights/insights-runtime-extractor-8hnkh] addLogicalPort took 17.536802ms, libovsdb time 686.433µs I0326 23:12:54.032245 3952 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-hzh6s] creating logical port openshift-dns_dns-default-hzh6s for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:54.032337 3952 kube.go:256] Updating pod openshift-dns/dns-default-hzh6s I0326 23:12:54.052945 3952 pod.go:62] [openshift-dns/dns-default-hzh6s] pod update took 20.630276ms I0326 23:12:54.052967 3952 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-hzh6s] addLogicalPort annotation time took 20.659478ms I0326 23:12:54.054336 3952 pods.go:271] [openshift-dns/dns-default-hzh6s] addLogicalPort took 22.106192ms, libovsdb time 1.005924ms I0326 23:12:54.246997 3952 cni.go:403] [openshift-image-registry/image-registry-7fc9774446-hnkrj f2030b9cbffb1c0ee9db2617643a59600ac239e43a84373383d1eb814d436490 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7fc9774446-hnkrj f2030b9cbffb1c0ee9db2617643a59600ac239e43a84373383d1eb814d436490 network default NAD default] I0326 23:12:54.269089 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7fc9774446-hnkrj, hostIfaceName: f2030b9cbffb1c0, network: default, NAD default, SandboxID: "f2030b9cbffb1c0ee9db2617643a59600ac239e43a84373383d1eb814d436490", PCI device ID: , UID: "616d75a7-0843-4c00-b5af-066754d5ff9e", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0326 23:12:54.313310 3952 cni.go:424] [openshift-image-registry/image-registry-7fc9774446-hnkrj f2030b9cbffb1c0ee9db2617643a59600ac239e43a84373383d1eb814d436490 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7fc9774446-hnkrj f2030b9cbffb1c0ee9db2617643a59600ac239e43a84373383d1eb814d436490 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:db:36:e6:69:56\",\"name\":\"f2030b9cbffb1c0\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e522d78-52b9-45ee-9368-8ac9030d69b0\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:54.333631 3952 cni.go:403] [openshift-ingress-canary/ingress-canary-kzq6d 51b4db7b58f1a54ff895b3f9db459e47e9c6ecaf2239bcef39086ca71853025c network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-kzq6d 51b4db7b58f1a54ff895b3f9db459e47e9c6ecaf2239bcef39086ca71853025c network default NAD default] I0326 23:12:54.344557 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-kzq6d, hostIfaceName: 51b4db7b58f1a54, network: default, NAD default, SandboxID: "51b4db7b58f1a54ff895b3f9db459e47e9c6ecaf2239bcef39086ca71853025c", PCI device ID: , UID: "2053308b-f0c6-45e6-8e3d-e1aba63ddd24", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0326 23:12:54.348416 3952 cni.go:403] [openshift-insights/insights-runtime-extractor-8hnkh b6d4facf87b2c8e3018a32bd0e57d08d0a1d780deea7e55ecdf12bc47c7df7be network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-8hnkh b6d4facf87b2c8e3018a32bd0e57d08d0a1d780deea7e55ecdf12bc47c7df7be network default NAD default] I0326 23:12:54.376167 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-8hnkh, hostIfaceName: b6d4facf87b2c8e, network: default, NAD default, SandboxID: "b6d4facf87b2c8e3018a32bd0e57d08d0a1d780deea7e55ecdf12bc47c7df7be", PCI device ID: , UID: "ddb560ea-4512-48ca-93f9-092f31553381", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0326 23:12:54.405209 3952 cni.go:424] [openshift-ingress-canary/ingress-canary-kzq6d 51b4db7b58f1a54ff895b3f9db459e47e9c6ecaf2239bcef39086ca71853025c network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-kzq6d 51b4db7b58f1a54ff895b3f9db459e47e9c6ecaf2239bcef39086ca71853025c network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:bd:66:44:66:45\",\"name\":\"51b4db7b58f1a54\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f208e4f0-8179-4858-b9e3-f03ba52c1fa3\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:54.419972 3952 cni.go:424] [openshift-insights/insights-runtime-extractor-8hnkh b6d4facf87b2c8e3018a32bd0e57d08d0a1d780deea7e55ecdf12bc47c7df7be network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-8hnkh b6d4facf87b2c8e3018a32bd0e57d08d0a1d780deea7e55ecdf12bc47c7df7be network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:46:ab:38:0f:a2\",\"name\":\"b6d4facf87b2c8e\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/59c28b67-7e7b-442f-9a9b-305cf09d3d62\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:54.438331 3952 cni.go:403] [openshift-dns/dns-default-hzh6s 01b95504675603a7b1955832c42a57a36caf7cc1df9f316364a12a37bddea7c4 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-hzh6s 01b95504675603a7b1955832c42a57a36caf7cc1df9f316364a12a37bddea7c4 network default NAD default] I0326 23:12:54.448169 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-hzh6s, hostIfaceName: 01b95504675603a, network: default, NAD default, SandboxID: "01b95504675603a7b1955832c42a57a36caf7cc1df9f316364a12a37bddea7c4", PCI device ID: , UID: "542f3f30-c910-4bb9-9fb8-93d80b25cade", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0326 23:12:54.493700 3952 cni.go:424] [openshift-dns/dns-default-hzh6s 01b95504675603a7b1955832c42a57a36caf7cc1df9f316364a12a37bddea7c4 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-hzh6s 01b95504675603a7b1955832c42a57a36caf7cc1df9f316364a12a37bddea7c4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:78:f5:36:64:8a\",\"name\":\"01b95504675603a\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67332e53-6e40-4f2a-bdfe-faf7db6e6716\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:12:55.341209 3952 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:12:55.435244 3952 base_network_controller_pods.go:487] [default/openshift-console/console-cfc8b65f6-w8l4f] creating logical port openshift-console_console-cfc8b65f6-w8l4f for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:55.435397 3952 kube.go:256] Updating pod openshift-console/console-cfc8b65f6-w8l4f I0326 23:12:55.450664 3952 pod.go:62] [openshift-console/console-cfc8b65f6-w8l4f] pod update took 15.295977ms I0326 23:12:55.450688 3952 base_network_controller_pods.go:937] [default/openshift-console/console-cfc8b65f6-w8l4f] addLogicalPort annotation time took 15.328214ms I0326 23:12:55.452057 3952 pods.go:271] [openshift-console/console-cfc8b65f6-w8l4f] addLogicalPort took 16.835669ms, libovsdb time 1.060921ms W0326 23:12:56.479070 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:12:56.825674 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:12:56.831241 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2] creating logical port openshift-monitoring_openshift-state-metrics-bc865d6bb-4qpw2 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:56.831354 3952 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2 I0326 23:12:56.849211 3952 pod.go:62] [openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2] pod update took 17.879391ms I0326 23:12:56.849230 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2] addLogicalPort annotation time took 17.902607ms I0326 23:12:56.850372 3952 pods.go:271] [openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2] addLogicalPort took 19.141041ms, libovsdb time 854.41µs I0326 23:12:56.891625 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7] creating logical port openshift-monitoring_kube-state-metrics-664d8f7fc8-45nf7 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:56.891704 3952 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7 I0326 23:12:56.913379 3952 pod.go:62] [openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7] pod update took 21.695815ms I0326 23:12:56.913404 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7] addLogicalPort annotation time took 21.724434ms I0326 23:12:56.914521 3952 pods.go:271] [openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7] addLogicalPort took 22.909056ms, libovsdb time 864.065µs I0326 23:12:57.403401 3952 cni.go:403] [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default] ADD starting CNI request [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default] I0326 23:12:57.405680 3952 cni.go:403] [openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2 1d818b8288cb10efd42f3a69c21526eec55b9ade016b7c49d65bd1ed4271015b network default NAD default] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2 1d818b8288cb10efd42f3a69c21526eec55b9ade016b7c49d65bd1ed4271015b network default NAD default] I0326 23:12:57.407274 3952 cni.go:403] [openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7 addb65d6a92a761144003c486468a67b4e4b3aedace0e38e9345dac72b35ffe6 network default NAD default] ADD starting CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7 addb65d6a92a761144003c486468a67b4e4b3aedace0e38e9345dac72b35ffe6 network default NAD default] I0326 23:12:57.416690 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-cfc8b65f6-w8l4f, hostIfaceName: 68c6d1151595e26, network: default, NAD default, SandboxID: "68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9", PCI device ID: , UID: "e8500e75-362e-40ba-ae08-c5ddcad6eb42", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0326 23:12:57.417471 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-bc865d6bb-4qpw2, hostIfaceName: 1d818b8288cb10e, network: default, NAD default, SandboxID: "1d818b8288cb10efd42f3a69c21526eec55b9ade016b7c49d65bd1ed4271015b", PCI device ID: , UID: "864907a1-7564-48eb-b12b-e6abc4454cb0", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0326 23:12:57.418024 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-664d8f7fc8-45nf7, hostIfaceName: addb65d6a92a761, network: default, NAD default, SandboxID: "addb65d6a92a761144003c486468a67b4e4b3aedace0e38e9345dac72b35ffe6", PCI device ID: , UID: "c0b16d9a-aa1d-46e8-880d-e5eef5143ccb", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0326 23:12:57.470644 3952 cni.go:424] [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default] ADD finished CNI request [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:41:7e:7c:ed:d2\",\"name\":\"68c6d1151595e26\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dbcc6513-b105-4295-9b2e-88b34fa4a08c\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:57.692099 3952 cni.go:424] [openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2 1d818b8288cb10efd42f3a69c21526eec55b9ade016b7c49d65bd1ed4271015b network default NAD default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-bc865d6bb-4qpw2 1d818b8288cb10efd42f3a69c21526eec55b9ade016b7c49d65bd1ed4271015b network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:8b:fd:a1:53:24\",\"name\":\"1d818b8288cb10e\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db8f41c9-3e0e-485e-91fc-7781f2fd2457\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:57.692114 3952 cni.go:424] [openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7 addb65d6a92a761144003c486468a67b4e4b3aedace0e38e9345dac72b35ffe6 network default NAD default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-45nf7 addb65d6a92a761144003c486468a67b4e4b3aedace0e38e9345dac72b35ffe6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:39:22:95:da:d9\",\"name\":\"addb65d6a92a761\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f4ce1903-d75d-469f-b1ff-6aeb81cd64df\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:57.835390 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:57.835498 3952 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0326 23:12:57.860082 3952 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 24.621339ms I0326 23:12:57.860108 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 24.65312ms I0326 23:12:57.861187 3952 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 25.811556ms, libovsdb time 804.907µs I0326 23:12:58.234685 3952 cni.go:403] [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default] I0326 23:12:58.247310 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 2bfa77a2f74a210, network: default, NAD default, SandboxID: "2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986", PCI device ID: , UID: "a526ed21-1861-4ce4-98a0-ccc934ecb924", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0326 23:12:58.300332 3952 cni.go:424] [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:83:74:b0:d3:cc\",\"name\":\"2bfa77a2f74a210\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5077275d-19ce-42d7-92fb-613be44fd000\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:12:59.674306 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-d9474544b-vhhxn] creating logical port openshift-monitoring_thanos-querier-d9474544b-vhhxn for pod on switch ip-10-0-0-90.ec2.internal I0326 23:12:59.674411 3952 kube.go:256] Updating pod openshift-monitoring/thanos-querier-d9474544b-vhhxn I0326 23:12:59.690961 3952 pod.go:62] [openshift-monitoring/thanos-querier-d9474544b-vhhxn] pod update took 16.58367ms I0326 23:12:59.690981 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-d9474544b-vhhxn] addLogicalPort annotation time took 16.607418ms I0326 23:12:59.692131 3952 pods.go:271] [openshift-monitoring/thanos-querier-d9474544b-vhhxn] addLogicalPort took 17.842363ms, libovsdb time 868.105µs I0326 23:13:01.089108 3952 cni.go:403] [openshift-monitoring/thanos-querier-d9474544b-vhhxn 263e911f5a2a83e6562dd5b99d8c993156e9464d887034396037415a8681af55 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-d9474544b-vhhxn 263e911f5a2a83e6562dd5b99d8c993156e9464d887034396037415a8681af55 network default NAD default] I0326 23:13:01.101807 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-d9474544b-vhhxn, hostIfaceName: 263e911f5a2a83e, network: default, NAD default, SandboxID: "263e911f5a2a83e6562dd5b99d8c993156e9464d887034396037415a8681af55", PCI device ID: , UID: "63956c6d-a5ea-4c12-8a03-0c3d930960a8", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0326 23:13:01.145831 3952 cni.go:424] [openshift-monitoring/thanos-querier-d9474544b-vhhxn 263e911f5a2a83e6562dd5b99d8c993156e9464d887034396037415a8681af55 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-d9474544b-vhhxn 263e911f5a2a83e6562dd5b99d8c993156e9464d887034396037415a8681af55 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:ca:f8:31:d5:98\",\"name\":\"263e911f5a2a83e\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/778065b6-7c35-4a68-b774-aace0e890f52\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:13:01.343578 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-789f77f87-q8g6k] creating logical port openshift-monitoring_metrics-server-789f77f87-q8g6k for pod on switch ip-10-0-0-90.ec2.internal I0326 23:13:01.343674 3952 kube.go:256] Updating pod openshift-monitoring/metrics-server-789f77f87-q8g6k I0326 23:13:01.378408 3952 pod.go:62] [openshift-monitoring/metrics-server-789f77f87-q8g6k] pod update took 34.7597ms I0326 23:13:01.378431 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-789f77f87-q8g6k] addLogicalPort annotation time took 34.789158ms I0326 23:13:01.379616 3952 pods.go:271] [openshift-monitoring/metrics-server-789f77f87-q8g6k] addLogicalPort took 36.057368ms, libovsdb time 865.676µs W0326 23:13:01.455523 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:13:01.455633 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:13:01.455637 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:13:01.509815 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-pm6sc for pod on switch ip-10-0-0-90.ec2.internal I0326 23:13:01.509966 3952 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc W0326 23:13:01.510058 3952 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-cfc8b65f6-9jpvm IP address from the namespace address-set, err: pod openshift-console/console-cfc8b65f6-9jpvm: no pod IPs found I0326 23:13:01.585515 3952 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc] pod update took 75.581058ms I0326 23:13:01.585540 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc] addLogicalPort annotation time took 75.613232ms I0326 23:13:01.586945 3952 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc] addLogicalPort took 77.209793ms, libovsdb time 1.02539ms I0326 23:13:02.447158 3952 cni.go:403] [openshift-monitoring/metrics-server-789f77f87-q8g6k 1b3acd5a19e9b4e4eacf1eb991ce31c31cc328390d3adc09996e8b2db1b3e195 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-789f77f87-q8g6k 1b3acd5a19e9b4e4eacf1eb991ce31c31cc328390d3adc09996e8b2db1b3e195 network default NAD default] I0326 23:13:02.453389 3952 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc d04fc8d6c966dc2ae4722821187063b63f0a31e7e74a3cea3d38ee3a4849a9d8 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc d04fc8d6c966dc2ae4722821187063b63f0a31e7e74a3cea3d38ee3a4849a9d8 network default NAD default] I0326 23:13:02.461666 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-789f77f87-q8g6k, hostIfaceName: 1b3acd5a19e9b4e, network: default, NAD default, SandboxID: "1b3acd5a19e9b4e4eacf1eb991ce31c31cc328390d3adc09996e8b2db1b3e195", PCI device ID: , UID: "b014ec74-6f7c-47fd-a1ad-28d947fd9c8f", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0326 23:13:02.465482 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-pm6sc, hostIfaceName: d04fc8d6c966dc2, network: default, NAD default, SandboxID: "d04fc8d6c966dc2ae4722821187063b63f0a31e7e74a3cea3d38ee3a4849a9d8", PCI device ID: , UID: "79259b28-cb3b-4588-a34c-a1204158634a", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0326 23:13:02.514932 3952 cni.go:424] [openshift-monitoring/metrics-server-789f77f87-q8g6k 1b3acd5a19e9b4e4eacf1eb991ce31c31cc328390d3adc09996e8b2db1b3e195 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-789f77f87-q8g6k 1b3acd5a19e9b4e4eacf1eb991ce31c31cc328390d3adc09996e8b2db1b3e195 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:24:ab:29:9f:60\",\"name\":\"1b3acd5a19e9b4e\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f073a09-f93f-4c85-93e0-47963543d256\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:13:02.535816 3952 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc d04fc8d6c966dc2ae4722821187063b63f0a31e7e74a3cea3d38ee3a4849a9d8 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-pm6sc d04fc8d6c966dc2ae4722821187063b63f0a31e7e74a3cea3d38ee3a4849a9d8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:36:a7:ca:ac:66\",\"name\":\"d04fc8d6c966dc2\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f4192e27-470e-43ad-b3b5-e1a8a36e7f77\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:13:02.982131 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:13:02.982256 3952 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0326 23:13:03.005812 3952 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 23.60545ms I0326 23:13:03.005837 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 23.635713ms I0326 23:13:03.007586 3952 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 25.476284ms, libovsdb time 1.400582ms W0326 23:13:03.424214 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:13:03.465315 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:03.465339 3952 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:03.465347 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:13:05.070891 3952 cni.go:403] [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default] I0326 23:13:05.081525 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: eb2568adfbac925, network: default, NAD default, SandboxID: "eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e", PCI device ID: , UID: "9472cae3-93a0-4d43-8011-4488de6eb3d9", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0326 23:13:05.127096 3952 cni.go:424] [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:77:a0:cd:ac:a2\",\"name\":\"eb2568adfbac925\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/97256904-c47e-4f9b-889f-16483645d87e\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:13:05.414094 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:13:05.517686 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:13:05.830015 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:13:06.429044 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0326 23:13:06.454859 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0326 23:13:06.499069 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:13:06.499107 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:13:06.572406 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:06.572426 3952 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:06.572431 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:08.375350 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:13:08.375375 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:13:08.375383 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:13:08.486672 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:08.486790 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:08.486801 3952 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:08.486806 3952 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:09.872646 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:13:09.937099 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:13:09.937118 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:13:10.470546 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:10.470573 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:10.470580 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:13:10.470587 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:13:16.199762 3952 obj_retry.go:464] Detected object openshift-image-registry/image-registry-bf569b496-g4fz4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:16.200428 3952 obj_retry.go:464] Detected object openshift-image-registry/image-registry-bf569b496-g4fz4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:13:16.362760 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:13:16.968741 3952 obj_retry.go:458] Detected object openshift-image-registry/image-registry-bf569b496-g4fz4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:16.968792 3952 obj_retry.go:458] Detected object openshift-image-registry/image-registry-bf569b496-g4fz4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:16.975891 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-bf569b496-g4fz4 I0326 23:13:16.975913 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-bf569b496-g4fz4 I0326 23:13:18.086547 3952 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:18.087096 3952 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:18.087123 3952 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:13:18.087132 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0326 23:13:18.087147 3952 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:13:18.087156 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0326 23:13:19.125262 3952 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:19.125291 3952 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:19.125305 3952 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:19.125318 3952 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mwj45 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:19.152003 3952 cni.go:403] [openshift-network-diagnostics/network-check-target-wmdfm d0ee622f0c7fe1c745810ec095856e736eaef74f94c7b35b792e2ccda2e4c2fd network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-wmdfm d0ee622f0c7fe1c745810ec095856e736eaef74f94c7b35b792e2ccda2e4c2fd network default NAD default] I0326 23:13:19.160881 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-wmdfm, hostIfaceName: d0ee622f0c7fe1c, network: default, NAD default, SandboxID: "d0ee622f0c7fe1c745810ec095856e736eaef74f94c7b35b792e2ccda2e4c2fd", PCI device ID: , UID: "a81dcdb5-cef2-4da4-958e-51c85bc5b032", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0326 23:13:19.189270 3952 cni.go:403] [openshift-multus/network-metrics-daemon-snttp 66ba1c57a030ba97f9774f0bb29a553d8f71bf6454d69d087d742923713212e1 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-snttp 66ba1c57a030ba97f9774f0bb29a553d8f71bf6454d69d087d742923713212e1 network default NAD default] I0326 23:13:19.199909 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-snttp, hostIfaceName: 66ba1c57a030ba9, network: default, NAD default, SandboxID: "66ba1c57a030ba97f9774f0bb29a553d8f71bf6454d69d087d742923713212e1", PCI device ID: , UID: "4ddf81a6-43c3-4f3c-93e7-3496c7edfd39", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0326 23:13:19.208730 3952 cni.go:424] [openshift-network-diagnostics/network-check-target-wmdfm d0ee622f0c7fe1c745810ec095856e736eaef74f94c7b35b792e2ccda2e4c2fd network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-wmdfm d0ee622f0c7fe1c745810ec095856e736eaef74f94c7b35b792e2ccda2e4c2fd network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:06:aa:5c:e5:f5\",\"name\":\"d0ee622f0c7fe1c\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/12aca702-f271-4a9d-81df-bc10b5f26745\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:13:19.245369 3952 cni.go:424] [openshift-multus/network-metrics-daemon-snttp 66ba1c57a030ba97f9774f0bb29a553d8f71bf6454d69d087d742923713212e1 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-snttp 66ba1c57a030ba97f9774f0bb29a553d8f71bf6454d69d087d742923713212e1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:86:5e:a3:41:91\",\"name\":\"66ba1c57a030ba9\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d53ea738-5072-41cb-9fa7-4b2f42780768\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:13:21.761204 3952 obj_retry.go:464] Detected object openshift-console/console-5767c898f4-rxqp5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:21.761758 3952 obj_retry.go:464] Detected object openshift-console/console-5767c898f4-rxqp5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:22.615955 3952 obj_retry.go:458] Detected object openshift-console/console-5767c898f4-rxqp5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:22.615981 3952 obj_retry.go:458] Detected object openshift-console/console-5767c898f4-rxqp5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:22.623639 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5767c898f4-rxqp5 I0326 23:13:22.623665 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5767c898f4-rxqp5 I0326 23:13:23.920698 3952 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:13:23.932442 3952 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0326 23:13:26.438639 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0326 23:13:26.725603 3952 obj_retry.go:464] Detected object openshift-console/console-5767c898f4-5dkfm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:26.726178 3952 obj_retry.go:464] Detected object openshift-console/console-5767c898f4-5dkfm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:26.970293 3952 obj_retry.go:458] Detected object openshift-console/console-5767c898f4-5dkfm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:26.970316 3952 obj_retry.go:458] Detected object openshift-console/console-5767c898f4-5dkfm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:26.975828 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5767c898f4-5dkfm I0326 23:13:26.975850 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5767c898f4-5dkfm W0326 23:13:28.249129 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:28.249151 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:28.249157 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:13:39.038722 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:13:40.622892 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:13:40.622913 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:13:41.730672 3952 obj_retry.go:464] Detected object openshift-image-registry/image-registry-646b7b7b8-hzth5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:41.731244 3952 obj_retry.go:464] Detected object openshift-image-registry/image-registry-646b7b7b8-hzth5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:42.671251 3952 obj_retry.go:458] Detected object openshift-image-registry/image-registry-646b7b7b8-hzth5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:42.671287 3952 obj_retry.go:458] Detected object openshift-image-registry/image-registry-646b7b7b8-hzth5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:42.677562 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-646b7b7b8-hzth5 I0326 23:13:42.677586 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-646b7b7b8-hzth5 W0326 23:13:45.070070 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:13:49.092954 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0326 23:13:54.524218 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:13:57.846401 3952 cni.go:403] [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default] DEL starting CNI request [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default] I0326 23:13:57.920433 3952 cni.go:424] [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default] DEL finished CNI request [openshift-console/console-cfc8b65f6-w8l4f 68c6d1151595e26ffcb2929ffcc78cf132563680b7f010d044acbec44db87ef9 network default NAD default], result "{}", err I0326 23:13:57.963600 3952 obj_retry.go:464] Detected object openshift-console/console-cfc8b65f6-w8l4f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:57.963649 3952 pods.go:176] Deleting pod: openshift-console/console-cfc8b65f6-w8l4f I0326 23:13:57.967193 3952 pods.go:236] Attempting to release IPs for pod: openshift-console/console-cfc8b65f6-w8l4f, ips: 10.130.0.9 I0326 23:13:57.967227 3952 obj_retry.go:464] Detected object openshift-console/console-cfc8b65f6-w8l4f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:13:58.648155 3952 obj_retry.go:458] Detected object openshift-console/console-cfc8b65f6-w8l4f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:58.648178 3952 obj_retry.go:458] Detected object openshift-console/console-cfc8b65f6-w8l4f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:13:58.651178 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-cfc8b65f6-w8l4f I0326 23:13:58.651202 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-cfc8b65f6-w8l4f I0326 23:14:18.485075 3952 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:18.485647 3952 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:18.824190 3952 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:18.824212 3952 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:18.829621 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0326 23:14:18.829646 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0326 23:14:22.891864 3952 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:22.892550 3952 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:23.256213 3952 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:23.256241 3952 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:23.262864 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0326 23:14:23.262889 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0326 23:14:25.059513 3952 base_network_controller_pods.go:487] [default/openshift-console/console-dfdb5bcfd-dwg7g] creating logical port openshift-console_console-dfdb5bcfd-dwg7g for pod on switch ip-10-0-0-90.ec2.internal I0326 23:14:25.059614 3952 kube.go:256] Updating pod openshift-console/console-dfdb5bcfd-dwg7g I0326 23:14:25.091031 3952 pod.go:62] [openshift-console/console-dfdb5bcfd-dwg7g] pod update took 31.439211ms I0326 23:14:25.091061 3952 base_network_controller_pods.go:937] [default/openshift-console/console-dfdb5bcfd-dwg7g] addLogicalPort annotation time took 31.473581ms I0326 23:14:25.092415 3952 pods.go:271] [openshift-console/console-dfdb5bcfd-dwg7g] addLogicalPort took 32.91906ms, libovsdb time 972.531µs I0326 23:14:25.415286 3952 cni.go:403] [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default] ADD starting CNI request [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default] I0326 23:14:25.423870 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-dfdb5bcfd-dwg7g, hostIfaceName: 694962b985e61ab, network: default, NAD default, SandboxID: "694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59", PCI device ID: , UID: "57e66676-b9c9-4377-ad87-9e91a959ff9e", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0326 23:14:25.469115 3952 cni.go:424] [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default] ADD finished CNI request [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:f1:48:71:04:15\",\"name\":\"694962b985e61ab\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e9e753f-48e9-43e6-83e6-80c78491bbd8\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:14:35.811038 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:14:40.445898 3952 cni.go:403] [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default] I0326 23:14:40.524515 3952 cni.go:424] [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 2bfa77a2f74a210887d8e61f97e89fbf68739a571c6af7b551c64f1653789986 network default NAD default], result "{}", err I0326 23:14:40.584071 3952 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:40.584184 3952 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0326 23:14:40.585727 3952 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.130.0.12 I0326 23:14:40.585756 3952 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:40.830379 3952 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:40.830408 3952 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:40.836697 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0326 23:14:40.836719 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0326 23:14:40.872313 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:14:40.872460 3952 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0326 23:14:40.904899 3952 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 32.50648ms I0326 23:14:40.904922 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 32.534232ms I0326 23:14:40.909182 3952 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 36.888945ms, libovsdb time 3.920229ms I0326 23:14:41.230946 3952 cni.go:403] [openshift-monitoring/alertmanager-main-0 a87945ce1079b45b556403cfa429db796b0924d1264f062371ff5df8054483fc network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 a87945ce1079b45b556403cfa429db796b0924d1264f062371ff5df8054483fc network default NAD default] I0326 23:14:41.239434 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: a87945ce1079b45, network: default, NAD default, SandboxID: "a87945ce1079b45b556403cfa429db796b0924d1264f062371ff5df8054483fc", PCI device ID: , UID: "8c964a24-480a-49ec-b5b4-4e3ebb62e846", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0326 23:14:41.283798 3952 cni.go:424] [openshift-monitoring/alertmanager-main-0 a87945ce1079b45b556403cfa429db796b0924d1264f062371ff5df8054483fc network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 a87945ce1079b45b556403cfa429db796b0924d1264f062371ff5df8054483fc network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:e3:be:94:e3:1f\",\"name\":\"a87945ce1079b45\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4902f140-239a-40b0-bdf8-29c4a92a5b05\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:14:50.356003 3952 obj_retry.go:464] Detected object openshift-console/console-755566f549-cdp5p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:50.356544 3952 obj_retry.go:464] Detected object openshift-console/console-755566f549-cdp5p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:14:50.965175 3952 obj_retry.go:458] Detected object openshift-console/console-755566f549-cdp5p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:50.965209 3952 obj_retry.go:458] Detected object openshift-console/console-755566f549-cdp5p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:14:50.971710 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-755566f549-cdp5p I0326 23:14:50.971732 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-755566f549-cdp5p I0326 23:15:01.140510 3952 obj_retry.go:464] Detected object openshift-console/console-755566f549-n8kgm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:01.141113 3952 obj_retry.go:464] Detected object openshift-console/console-755566f549-n8kgm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:15:01.295106 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:15:01.295134 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:15:01.295142 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:15:01.415095 3952 obj_retry.go:458] Detected object openshift-console/console-755566f549-n8kgm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:01.415127 3952 obj_retry.go:458] Detected object openshift-console/console-755566f549-n8kgm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:01.420589 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-755566f549-n8kgm I0326 23:15:01.420612 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-755566f549-n8kgm I0326 23:15:13.161959 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:13.162552 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:14.055269 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:14.055295 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:25.751189 3952 cni.go:403] [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default] I0326 23:15:25.821119 3952 cni.go:424] [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 eb2568adfbac9257b17e00013b17f211ef49de599f17938b1344cc97383dcd6e network default NAD default], result "{}", err I0326 23:15:25.879614 3952 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:25.879658 3952 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0326 23:15:25.880994 3952 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.130.0.16 I0326 23:15:25.881022 3952 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:26.015249 3952 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:26.015276 3952 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:26.021945 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0326 23:15:26.022030 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0326 23:15:26.065749 3952 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:15:26.065924 3952 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0326 23:15:26.093601 3952 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 27.719299ms I0326 23:15:26.093629 3952 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 27.753797ms I0326 23:15:26.095118 3952 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 29.390563ms, libovsdb time 944.906µs I0326 23:15:26.428103 3952 cni.go:403] [openshift-monitoring/prometheus-k8s-0 63230e48fb42b4519be17b3d0bedb02c6f09ad03dd63c5c08fe7be9b3992d0d5 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 63230e48fb42b4519be17b3d0bedb02c6f09ad03dd63c5c08fe7be9b3992d0d5 network default NAD default] I0326 23:15:26.437172 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 63230e48fb42b45, network: default, NAD default, SandboxID: "63230e48fb42b4519be17b3d0bedb02c6f09ad03dd63c5c08fe7be9b3992d0d5", PCI device ID: , UID: "80b9dfc1-02ed-4f01-8586-4e56c66febf1", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0326 23:15:26.480638 3952 cni.go:424] [openshift-monitoring/prometheus-k8s-0 63230e48fb42b4519be17b3d0bedb02c6f09ad03dd63c5c08fe7be9b3992d0d5 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 63230e48fb42b4519be17b3d0bedb02c6f09ad03dd63c5c08fe7be9b3992d0d5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:b7:bd:f6:5e:56\",\"name\":\"63230e48fb42b45\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33d1e85e-dfc9-4733-b343-9f4b088648ed\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:16:27.296600 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:16:27.296622 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:16:59.816386 3952 obj_retry.go:464] Detected object openshift-console/console-dfdb5bcfd-s54ll of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:59.817005 3952 obj_retry.go:464] Detected object openshift-console/console-dfdb5bcfd-s54ll of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:00.458144 3952 obj_retry.go:458] Detected object openshift-console/console-dfdb5bcfd-s54ll of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:00.458168 3952 obj_retry.go:458] Detected object openshift-console/console-dfdb5bcfd-s54ll of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:00.463980 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-dfdb5bcfd-s54ll I0326 23:17:00.464002 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-dfdb5bcfd-s54ll I0326 23:17:10.087097 3952 cni.go:403] [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default] DEL starting CNI request [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default] I0326 23:17:10.156986 3952 cni.go:424] [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default] DEL finished CNI request [openshift-console/console-dfdb5bcfd-dwg7g 694962b985e61ab42774989ee272478ed0271834027fff566839460cf7429e59 network default NAD default], result "{}", err I0326 23:17:10.203852 3952 obj_retry.go:464] Detected object openshift-console/console-dfdb5bcfd-dwg7g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:10.203913 3952 pods.go:176] Deleting pod: openshift-console/console-dfdb5bcfd-dwg7g I0326 23:17:10.205330 3952 pods.go:236] Attempting to release IPs for pod: openshift-console/console-dfdb5bcfd-dwg7g, ips: 10.130.0.17 I0326 23:17:10.205361 3952 obj_retry.go:464] Detected object openshift-console/console-dfdb5bcfd-dwg7g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:10.421506 3952 obj_retry.go:458] Detected object openshift-console/console-dfdb5bcfd-dwg7g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:10.421535 3952 obj_retry.go:458] Detected object openshift-console/console-dfdb5bcfd-dwg7g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:10.424815 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-dfdb5bcfd-dwg7g I0326 23:17:10.424838 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-dfdb5bcfd-dwg7g I0326 23:17:40.787566 3952 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:18:01.023487 3952 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 W0326 23:18:28.774332 3952 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:18:39.812217 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:18:39.812278 3952 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-sptr7 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-sptr7: no pod IPs found I0326 23:18:48.821386 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0326 23:18:58.013174 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0326 23:19:45.574939 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=11 I0326 23:19:52.570712 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=425 I0326 23:19:56.920191 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0326 23:20:04.248813 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:04.249438 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:05.145008 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:05.145069 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:09.599355 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0326 23:20:39.574737 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=34 I0326 23:20:41.383806 3952 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-90.ec2.internal I0326 23:20:41.383844 3952 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-90.ec2.internal: 45.828µs I0326 23:20:41.383799 3952 egressfirewall.go:628] Syncing node "ip-10-0-0-90.ec2.internal" for egress firewall I0326 23:20:41.411464 3952 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-p44j6] creating logical port kube-system_global-pull-secret-syncer-p44j6 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:20:41.411607 3952 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-p44j6 I0326 23:20:41.424793 3952 pod.go:62] [kube-system/global-pull-secret-syncer-p44j6] pod update took 13.204657ms I0326 23:20:41.424814 3952 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-p44j6] addLogicalPort annotation time took 13.252704ms I0326 23:20:41.426065 3952 pods.go:271] [kube-system/global-pull-secret-syncer-p44j6] addLogicalPort took 14.620426ms, libovsdb time 886.898µs I0326 23:20:41.767893 3952 cni.go:403] [kube-system/global-pull-secret-syncer-p44j6 072297e783e746ff255f2822acd3a0a8d8a6c488d2e30c4e96c395817a4c27fe network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-p44j6 072297e783e746ff255f2822acd3a0a8d8a6c488d2e30c4e96c395817a4c27fe network default NAD default] I0326 23:20:41.777192 3952 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-p44j6, hostIfaceName: 072297e783e746f, network: default, NAD default, SandboxID: "072297e783e746ff255f2822acd3a0a8d8a6c488d2e30c4e96c395817a4c27fe", PCI device ID: , UID: "489edf02-dc44-47e1-a0ec-a2a85bf07012", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0326 23:20:41.820253 3952 cni.go:424] [kube-system/global-pull-secret-syncer-p44j6 072297e783e746ff255f2822acd3a0a8d8a6c488d2e30c4e96c395817a4c27fe network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-p44j6 072297e783e746ff255f2822acd3a0a8d8a6c488d2e30c4e96c395817a4c27fe network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:98:6b:15:f0:43\",\"name\":\"072297e783e746f\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/99435c32-4320-4ccf-bc58-be6e3ec0040b\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:20:50.113282 3952 namespace.go:98] [tsf] adding namespace I0326 23:20:50.114152 3952 namespace.go:102] [tsf] adding namespace took 846.186µs I0326 23:20:50.127726 3952 namespace.go:142] [tsf] updating namespace I0326 23:20:50.147792 3952 namespace.go:142] [tsf] updating namespace I0326 23:20:53.897208 3952 namespace.go:98] [tsf-keycloak] adding namespace I0326 23:20:53.898167 3952 namespace.go:102] [tsf-keycloak] adding namespace took 927.928µs I0326 23:20:53.900126 3952 namespace.go:98] [rhtpa-operator] adding namespace I0326 23:20:53.900185 3952 namespace.go:98] [konflux-ui] adding namespace I0326 23:20:53.900883 3952 namespace.go:102] [rhtpa-operator] adding namespace took 738.651µs I0326 23:20:53.901452 3952 namespace.go:102] [konflux-ui] adding namespace took 1.249533ms I0326 23:20:53.904443 3952 namespace.go:98] [rhbk-operator] adding namespace I0326 23:20:53.905081 3952 namespace.go:102] [rhbk-operator] adding namespace took 624.323µs I0326 23:20:53.905888 3952 namespace.go:98] [cert-manager-operator] adding namespace I0326 23:20:53.905994 3952 namespace.go:98] [tsf-tas] adding namespace I0326 23:20:53.906044 3952 namespace.go:98] [konflux-operator] adding namespace I0326 23:20:53.906062 3952 namespace.go:98] [tsf-tpa] adding namespace I0326 23:20:53.906627 3952 namespace.go:102] [cert-manager-operator] adding namespace took 725.787µs I0326 23:20:53.907171 3952 namespace.go:102] [tsf-tas] adding namespace took 1.163011ms I0326 23:20:53.907666 3952 namespace.go:102] [konflux-operator] adding namespace took 1.608341ms I0326 23:20:53.908159 3952 namespace.go:102] [tsf-tpa] adding namespace took 2.086457ms I0326 23:20:53.918622 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:20:53.939449 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:20:53.959944 3952 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:20:53.960005 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:20:53.960080 3952 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:20:53.963965 3952 namespace.go:142] [rhbk-operator] updating namespace I0326 23:20:53.967316 3952 namespace.go:142] [tsf-tas] updating namespace I0326 23:20:53.973004 3952 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:20:53.976874 3952 namespace.go:142] [konflux-operator] updating namespace I0326 23:20:53.979958 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:20:53.985741 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:20:53.987548 3952 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:20:54.044729 3952 namespace.go:142] [rhbk-operator] updating namespace I0326 23:20:54.057982 3952 namespace.go:142] [rhbk-operator] updating namespace I0326 23:20:54.077029 3952 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:20:54.096046 3952 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:20:54.115043 3952 namespace.go:142] [tsf-tas] updating namespace I0326 23:20:54.129012 3952 namespace.go:142] [tsf-tas] updating namespace I0326 23:20:54.160734 3952 namespace.go:142] [konflux-operator] updating namespace I0326 23:20:54.173014 3952 namespace.go:142] [konflux-operator] updating namespace I0326 23:20:54.191239 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:20:54.201453 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:20:54.208034 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:20:54.228362 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:20:55.004054 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:20:55.018482 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:20:55.162341 3952 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-0-90.ec2.internal I0326 23:20:55.162461 3952 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0326 23:20:55.177431 3952 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 14.993686ms I0326 23:20:55.177459 3952 base_network_controller_pods.go:937] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 15.025753ms I0326 23:20:55.178990 3952 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 16.662641ms, libovsdb time 1.207092ms I0326 23:20:55.539862 3952 cni.go:403] [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default] ADD starting CNI request [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default] I0326 23:20:55.547839 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 4eb1611314fc523, network: default, NAD default, SandboxID: "4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7", PCI device ID: , UID: "ab5de046-85c9-4881-9196-0cfff92dea4f", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0326 23:20:55.590924 3952 cni.go:424] [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default] ADD finished CNI request [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:bf:df:89:9d:e4\",\"name\":\"4eb1611314fc523\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/84e45554-e706-48c1-b082-81fd90967ff8\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:21:02.175437 3952 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:02.176066 3952 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:02.176095 3952 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:02.176103 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:21:02.176118 3952 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:02.176123 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:21:03.188649 3952 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:03.189272 3952 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:03.189303 3952 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:03.189311 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:21:03.189325 3952 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:03.189334 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:21:03.200874 3952 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:03.200895 3952 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:03.200905 3952 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:03.200919 3952 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8vfv5p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:04.191278 3952 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:04.191933 3952 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:04.191968 3952 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:04.191978 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:21:04.191991 3952 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:04.191995 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:21:04.212078 3952 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:04.212097 3952 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:04.212113 3952 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:04.212126 3952 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5f9z4p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:04.572876 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=61 I0326 23:21:05.213500 3952 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:05.213525 3952 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:05.213538 3952 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:05.213550 3952 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf9jl6m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:06.203743 3952 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:06.204373 3952 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:06.204410 3952 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:06.204419 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:21:06.204434 3952 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:06.204440 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:21:07.229091 3952 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:07.229123 3952 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:07.229141 3952 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:07.229159 3952 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5l67 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:08.223832 3952 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.224441 3952 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.224468 3952 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.224478 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:21:08.224491 3952 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.224497 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:21:08.263393 3952 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.264070 3952 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.264093 3952 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.264101 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:21:08.264119 3952 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:21:08.264123 3952 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:21:09.253742 3952 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.253797 3952 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.253819 3952 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.253835 3952 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jjwdp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.292094 3952 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.292123 3952 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.292142 3952 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:09.292153 3952 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf2bvkm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:21:13.393136 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0326 23:21:16.055848 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0326 23:21:16.062222 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:21:21.466169 3952 namespace.go:142] [openshift-operators] updating namespace I0326 23:21:21.479634 3952 namespace.go:142] [openshift-operators] updating namespace I0326 23:21:22.865526 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 W0326 23:21:23.475249 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:21:24.573429 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 W0326 23:21:27.817718 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:21:27.828516 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:21:27.848635 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:21:27.874553 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:21:29.425244 3952 namespace.go:142] [openshift-operators] updating namespace I0326 23:21:29.616472 3952 namespace.go:98] [cert-manager] adding namespace I0326 23:21:29.617254 3952 namespace.go:102] [cert-manager] adding namespace took 759.939µs I0326 23:21:29.655658 3952 namespace.go:142] [cert-manager] updating namespace I0326 23:21:29.684805 3952 namespace.go:142] [cert-manager] updating namespace W0326 23:21:35.535469 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:21:35.535490 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:21:39.415680 3952 namespace.go:98] [trusted-artifact-signer] adding namespace I0326 23:21:39.416524 3952 namespace.go:102] [trusted-artifact-signer] adding namespace took 821.199µs I0326 23:21:39.428659 3952 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:21:39.428750 3952 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:21:39.467847 3952 cni.go:403] [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default] DEL starting CNI request [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default] I0326 23:21:39.541046 3952 cni.go:424] [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default] DEL finished CNI request [tsf/test-tsf-subscriptions 4eb1611314fc523c10142da8cc9d7605dc6e636889ce2e85701b04bc6a8051c7 network default NAD default], result "{}", err W0326 23:21:39.623553 3952 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:21:39.839831 3952 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:39.839862 3952 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0326 23:21:39.840958 3952 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.130.0.21 I0326 23:21:39.840981 3952 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:21:39.862860 3952 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:39.862883 3952 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:21:39.900896 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0326 23:21:39.900919 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0326 23:21:39.937562 3952 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-mgqwl for pod on switch ip-10-0-0-90.ec2.internal I0326 23:21:39.937645 3952 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl I0326 23:21:39.982609 3952 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl] pod update took 44.984795ms I0326 23:21:39.982631 3952 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl] addLogicalPort annotation time took 45.013764ms I0326 23:21:39.983786 3952 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl] addLogicalPort took 46.241685ms, libovsdb time 804.431µs I0326 23:21:40.295942 3952 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl 33f6f66d263784fa783eed1f4d8a5e40fc1b35038548e3c71cb0df65243dd0a0 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl 33f6f66d263784fa783eed1f4d8a5e40fc1b35038548e3c71cb0df65243dd0a0 network default NAD default] I0326 23:21:40.305352 3952 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-mgqwl, hostIfaceName: 33f6f66d263784f, network: default, NAD default, SandboxID: "33f6f66d263784fa783eed1f4d8a5e40fc1b35038548e3c71cb0df65243dd0a0", PCI device ID: , UID: "5712ab2b-13c2-455a-84c1-9d458d7fd698", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0326 23:21:40.350985 3952 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl 33f6f66d263784fa783eed1f4d8a5e40fc1b35038548e3c71cb0df65243dd0a0 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-mgqwl 33f6f66d263784fa783eed1f4d8a5e40fc1b35038548e3c71cb0df65243dd0a0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:a0:94:dc:34:3f\",\"name\":\"33f6f66d263784f\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/038d5249-1091-4d09-9f1f-9242e4ecb30d\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:21:41.202752 3952 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-0-90.ec2.internal I0326 23:21:41.202877 3952 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0326 23:21:41.340918 3952 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 138.073379ms I0326 23:21:41.340943 3952 base_network_controller_pods.go:937] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 138.104393ms I0326 23:21:41.342052 3952 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 139.316716ms, libovsdb time 780.628µs I0326 23:21:41.569716 3952 cni.go:403] [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default] ADD starting CNI request [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default] I0326 23:21:41.580437 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 83a800632f8b171, network: default, NAD default, SandboxID: "83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd", PCI device ID: , UID: "da90a8e3-7e7a-4060-9e61-433c33a40c40", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0326 23:21:41.623642 3952 cni.go:424] [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default] ADD finished CNI request [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:2f:f6:4c:75:02\",\"name\":\"83a800632f8b171\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/df5cc733-3a0b-43dc-b414-c9cff146f211\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:21:44.231652 3952 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-brbbb for pod on switch ip-10-0-0-90.ec2.internal I0326 23:21:44.231821 3952 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb I0326 23:21:44.270666 3952 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb] pod update took 38.913291ms I0326 23:21:44.270705 3952 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb] addLogicalPort annotation time took 38.951512ms I0326 23:21:44.272324 3952 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb] addLogicalPort took 40.692672ms, libovsdb time 1.111443ms I0326 23:21:46.987460 3952 namespace.go:98] [openshift-pipelines] adding namespace I0326 23:21:46.988445 3952 namespace.go:102] [openshift-pipelines] adding namespace took 952.777µs I0326 23:21:47.022011 3952 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:21:47.842093 3952 namespace.go:142] [cert-manager] updating namespace I0326 23:21:48.023825 3952 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:21:48.419937 3952 namespace.go:142] [konflux-operator] updating namespace I0326 23:21:48.622500 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:21:48.790127 3952 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb 1c1f33775815d3beb415e78c4c6c8a75bc31242cea688021073946f40e71b6ad network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb 1c1f33775815d3beb415e78c4c6c8a75bc31242cea688021073946f40e71b6ad network default NAD default] I0326 23:21:48.800018 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-brbbb, hostIfaceName: 1c1f33775815d3b, network: default, NAD default, SandboxID: "1c1f33775815d3beb415e78c4c6c8a75bc31242cea688021073946f40e71b6ad", PCI device ID: , UID: "72ee686a-d4c9-46f1-abeb-4f3f49a12854", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0326 23:21:48.824961 3952 namespace.go:142] [open-cluster-management-2p9mn5g6ejev430bbu2pa8d8q491egfs] updating namespace I0326 23:21:48.843951 3952 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb 1c1f33775815d3beb415e78c4c6c8a75bc31242cea688021073946f40e71b6ad network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-brbbb 1c1f33775815d3beb415e78c4c6c8a75bc31242cea688021073946f40e71b6ad network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:9f:65:21:72:0f\",\"name\":\"1c1f33775815d3b\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cae6181-ee8c-41c6-a5da-f6304e8b9dc0\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:21:49.224661 3952 namespace.go:142] [rhbk-operator] updating namespace I0326 23:21:49.424993 3952 namespace.go:142] [rhtpa-operator] updating namespace W0326 23:21:49.512921 3952 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0326 23:21:49.626927 3952 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:21:49.829323 3952 namespace.go:142] [tsf] updating namespace I0326 23:21:50.028280 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:21:50.230845 3952 namespace.go:142] [tsf-tas] updating namespace I0326 23:21:50.432120 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:21:50.565305 3952 namespace.go:142] [cert-manager] updating namespace I0326 23:21:50.574023 3952 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:21:50.582685 3952 namespace.go:142] [default] updating namespace I0326 23:21:50.591844 3952 namespace.go:142] [konflux-operator] updating namespace I0326 23:21:50.600975 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:21:50.611112 3952 namespace.go:142] [open-cluster-management-2p9mn5g6ejev430bbu2pa8d8q491egfs] updating namespace I0326 23:21:50.620056 3952 namespace.go:142] [openshift] updating namespace I0326 23:21:50.630632 3952 namespace.go:142] [rhbk-operator] updating namespace I0326 23:21:50.641002 3952 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:21:50.648914 3952 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:21:50.657393 3952 namespace.go:142] [tsf] updating namespace I0326 23:21:50.669911 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:21:50.692313 3952 namespace.go:142] [tsf-tas] updating namespace I0326 23:21:50.713192 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:21:50.815675 3952 namespace.go:142] [cert-manager] updating namespace I0326 23:21:50.926201 3952 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:21:51.015948 3952 namespace.go:142] [default] updating namespace I0326 23:21:51.115114 3952 namespace.go:142] [konflux-operator] updating namespace I0326 23:21:51.215523 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:21:51.322621 3952 namespace.go:142] [open-cluster-management-2p9mn5g6ejev430bbu2pa8d8q491egfs] updating namespace I0326 23:21:51.431666 3952 namespace.go:142] [openshift] updating namespace I0326 23:21:51.546733 3952 namespace.go:142] [rhbk-operator] updating namespace I0326 23:21:51.642874 3952 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:21:51.745753 3952 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:21:51.825110 3952 namespace.go:142] [tsf] updating namespace I0326 23:21:51.921863 3952 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:21:52.021371 3952 namespace.go:142] [tsf-tas] updating namespace I0326 23:21:52.129811 3952 namespace.go:142] [tsf-tpa] updating namespace I0326 23:21:52.454693 3952 namespace.go:142] [openshift-pipelines] updating namespace W0326 23:21:56.371982 3952 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0326 23:21:59.572049 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=149 I0326 23:22:00.268323 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-j668n] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-j668n for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:00.268411 3952 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-j668n I0326 23:22:00.290030 3952 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-j668n] pod update took 21.640763ms I0326 23:22:00.290058 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-j668n] addLogicalPort annotation time took 21.670729ms I0326 23:22:00.291208 3952 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-j668n] addLogicalPort took 22.913647ms, libovsdb time 829.316µs I0326 23:22:00.345352 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-g2622 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:00.345446 3952 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622 I0326 23:22:00.361470 3952 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622] pod update took 16.05116ms I0326 23:22:00.361496 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622] addLogicalPort annotation time took 16.079731ms I0326 23:22:00.362828 3952 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622] addLogicalPort took 17.493008ms, libovsdb time 890.815µs I0326 23:22:00.632967 3952 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-j668n 3609d4d77300461defa65912a0e0dac5fab4245d20886db5d47c4e919c9ae66e network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-j668n 3609d4d77300461defa65912a0e0dac5fab4245d20886db5d47c4e919c9ae66e network default NAD default] I0326 23:22:00.641405 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-j668n, hostIfaceName: 3609d4d77300461, network: default, NAD default, SandboxID: "3609d4d77300461defa65912a0e0dac5fab4245d20886db5d47c4e919c9ae66e", PCI device ID: , UID: "afc6e880-4f96-4a0e-a433-5cd85fc3a076", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0326 23:22:00.687792 3952 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-j668n 3609d4d77300461defa65912a0e0dac5fab4245d20886db5d47c4e919c9ae66e network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-j668n 3609d4d77300461defa65912a0e0dac5fab4245d20886db5d47c4e919c9ae66e network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:8d:4e:72:90:df\",\"name\":\"3609d4d77300461\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f5f42241-852a-4cde-a67b-dde0e5756026\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:22:00.714228 3952 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622 13e6e9e30f72a03a1f05cf325369b83e3294888dd881017b589ec8dda6434450 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622 13e6e9e30f72a03a1f05cf325369b83e3294888dd881017b589ec8dda6434450 network default NAD default] I0326 23:22:00.723553 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-g2622, hostIfaceName: 13e6e9e30f72a03, network: default, NAD default, SandboxID: "13e6e9e30f72a03a1f05cf325369b83e3294888dd881017b589ec8dda6434450", PCI device ID: , UID: "d76163fb-1749-4854-bda6-3a14e7605e7a", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0326 23:22:00.769521 3952 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622 13e6e9e30f72a03a1f05cf325369b83e3294888dd881017b589ec8dda6434450 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-g2622 13e6e9e30f72a03a1f05cf325369b83e3294888dd881017b589ec8dda6434450 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:26:e9:80:fd:b0\",\"name\":\"13e6e9e30f72a03\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c2885ae-d50f-4bec-a946-f821ff5c7f13\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:22:04.404661 3952 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:22:04.404683 3952 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:22:04.404689 3952 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:22:04.688649 3952 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:22:04.688670 3952 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:22:04.688675 3952 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:22:04.688679 3952 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:22:04.803459 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 W0326 23:22:05.653038 3952 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:22:05.653059 3952 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:22:05.653064 3952 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0326 23:22:08.028871 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0326 23:22:09.029088 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0326 23:22:10.597410 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 W0326 23:22:11.525372 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0326 23:22:16.596684 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:22:16.596705 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:22:16.596711 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0326 23:22:23.492122 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-9kn46 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:23.492235 3952 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46 I0326 23:22:23.509606 3952 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46] pod update took 17.409921ms I0326 23:22:23.509628 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46] addLogicalPort annotation time took 17.436798ms I0326 23:22:23.510915 3952 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46] addLogicalPort took 18.81057ms, libovsdb time 958.873µs I0326 23:22:23.855174 3952 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46 13d25f7f2722b2edf6db96a60097bef0aa5099094bd281b34cf11b55f105cdcc network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46 13d25f7f2722b2edf6db96a60097bef0aa5099094bd281b34cf11b55f105cdcc network default NAD default] I0326 23:22:23.864223 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-9kn46, hostIfaceName: 13d25f7f2722b2e, network: default, NAD default, SandboxID: "13d25f7f2722b2edf6db96a60097bef0aa5099094bd281b34cf11b55f105cdcc", PCI device ID: , UID: "fcde8d63-7383-4883-b1ce-e512820d0a7f", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0326 23:22:23.908488 3952 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46 13d25f7f2722b2edf6db96a60097bef0aa5099094bd281b34cf11b55f105cdcc network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-9kn46 13d25f7f2722b2edf6db96a60097bef0aa5099094bd281b34cf11b55f105cdcc network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:b9:2e:fc:26:47\",\"name\":\"13d25f7f2722b2e\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f5ace0dd-8686-465c-8998-030a332054ad\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:22:26.522486 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0326 23:22:32.721983 3952 cni.go:403] [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default] DEL starting CNI request [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default] I0326 23:22:32.791410 3952 cni.go:424] [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default] DEL finished CNI request [tsf/test-tsf-infrastructure 83a800632f8b171100ee6cc310aa0292ef8346db084470b92a4109ead4fc77dd network default NAD default], result "{}", err I0326 23:22:32.839506 3952 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:22:32.839540 3952 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0326 23:22:32.840957 3952 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.130.0.23 I0326 23:22:32.840984 3952 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:22:32.859445 3952 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:22:32.859470 3952 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:22:32.865271 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0326 23:22:32.865294 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0326 23:22:33.848015 3952 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:33.848100 3952 kube.go:256] Updating pod tsf/test-tsf-iam I0326 23:22:33.862113 3952 pod.go:62] [tsf/test-tsf-iam] pod update took 14.038201ms I0326 23:22:33.862134 3952 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 14.063095ms I0326 23:22:33.863292 3952 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 15.292834ms, libovsdb time 820.437µs I0326 23:22:34.212424 3952 cni.go:403] [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default] I0326 23:22:34.220819 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 3ab4ebd5eececc6, network: default, NAD default, SandboxID: "3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3", PCI device ID: , UID: "4f120523-d8e5-481c-95cb-39a34bb62289", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0326 23:22:34.264943 3952 cni.go:424] [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:2d:d1:24:e3:a5\",\"name\":\"3ab4ebd5eececc6\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7f575d69-2149-44da-95c3-b30d229d914e\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:22:39.462817 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-xq7rl for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:39.462904 3952 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl I0326 23:22:39.478545 3952 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl] pod update took 15.659774ms I0326 23:22:39.478570 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl] addLogicalPort annotation time took 15.691247ms I0326 23:22:39.479794 3952 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl] addLogicalPort took 16.992651ms, libovsdb time 834.212µs I0326 23:22:39.822678 3952 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl 8ae52f56e271e21114b1127b7d2a0fa7d836d2537484f7ab8a8cd1086e359737 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl 8ae52f56e271e21114b1127b7d2a0fa7d836d2537484f7ab8a8cd1086e359737 network default NAD default] I0326 23:22:39.831623 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-xq7rl, hostIfaceName: 8ae52f56e271e21, network: default, NAD default, SandboxID: "8ae52f56e271e21114b1127b7d2a0fa7d836d2537484f7ab8a8cd1086e359737", PCI device ID: , UID: "035147c1-b18a-4520-b5d6-566fb99e1d1b", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] W0326 23:22:39.841907 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:22:39.841931 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:22:39.845013 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:22:39.876580 3952 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl 8ae52f56e271e21114b1127b7d2a0fa7d836d2537484f7ab8a8cd1086e359737 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-xq7rl 8ae52f56e271e21114b1127b7d2a0fa7d836d2537484f7ab8a8cd1086e359737 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:cb:9b:ff:62:e9\",\"name\":\"8ae52f56e271e21\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e1d097b-9a32-46c9-9d73-bb23d8629c73\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:22:44.728014 3952 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:22:47.797365 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:47.797493 3952 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0326 23:22:47.809114 3952 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 11.667203ms I0326 23:22:47.809144 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 11.706236ms I0326 23:22:47.810470 3952 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 13.126959ms, libovsdb time 910.885µs I0326 23:22:50.556851 3952 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 b3fb645823b7bf09510d9813ff285c2dee970072eb9b4f3e48e4c8d833c08752 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 b3fb645823b7bf09510d9813ff285c2dee970072eb9b4f3e48e4c8d833c08752 network default NAD default] I0326 23:22:50.565277 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: b3fb645823b7bf0, network: default, NAD default, SandboxID: "b3fb645823b7bf09510d9813ff285c2dee970072eb9b4f3e48e4c8d833c08752", PCI device ID: , UID: "da957dbc-5361-4838-a363-2d26c2f67cd2", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0326 23:22:50.610413 3952 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 b3fb645823b7bf09510d9813ff285c2dee970072eb9b4f3e48e4c8d833c08752 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 b3fb645823b7bf09510d9813ff285c2dee970072eb9b4f3e48e4c8d833c08752 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:59:58:13:de:e2\",\"name\":\"b3fb645823b7bf0\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/02db1c13-4f5a-45c8-b98c-9bf9902d041c\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:22:55.879741 3952 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:22:57.306935 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-6wwl4 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:22:57.307058 3952 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4 I0326 23:22:57.319919 3952 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4] pod update took 12.887472ms I0326 23:22:57.319971 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4] addLogicalPort annotation time took 12.955498ms I0326 23:22:57.321369 3952 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4] addLogicalPort took 14.458717ms, libovsdb time 1.000021ms I0326 23:22:58.268571 3952 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4 6640b54b07a5a33fdd1bce3712de295aef23394c489a8d74f5cd92900dec38cb network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4 6640b54b07a5a33fdd1bce3712de295aef23394c489a8d74f5cd92900dec38cb network default NAD default] I0326 23:22:58.277404 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-6wwl4, hostIfaceName: 6640b54b07a5a33, network: default, NAD default, SandboxID: "6640b54b07a5a33fdd1bce3712de295aef23394c489a8d74f5cd92900dec38cb", PCI device ID: , UID: "9e569c19-a042-4471-b876-031de96db013", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0326 23:22:58.530733 3952 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4 6640b54b07a5a33fdd1bce3712de295aef23394c489a8d74f5cd92900dec38cb network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-6wwl4 6640b54b07a5a33fdd1bce3712de295aef23394c489a8d74f5cd92900dec38cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:69:e0:07:00:80\",\"name\":\"6640b54b07a5a33\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92d3a57f-33b4-4c12-a57f-47e8838c628c\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:23:02.867300 3952 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:23:02.867320 3952 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:23:02.867325 3952 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0326 23:23:06.892018 3952 cni.go:403] [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default] I0326 23:23:06.969464 3952 cni.go:424] [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 3ab4ebd5eececc62254b7700834056bc33b2b4000a2d1db77653b3fab128f6b3 network default NAD default], result "{}", err I0326 23:23:07.019694 3952 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:07.019728 3952 pods.go:176] Deleting pod: tsf/test-tsf-iam I0326 23:23:07.021261 3952 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.130.0.28 I0326 23:23:07.021356 3952 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:07.055887 3952 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:07.055933 3952 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:07.061384 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:23:07.061411 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:23:07.757596 3952 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:07.757733 3952 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0326 23:23:07.770174 3952 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 12.474498ms I0326 23:23:07.770208 3952 base_network_controller_pods.go:937] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 12.510968ms I0326 23:23:07.771590 3952 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 14.012148ms, libovsdb time 1.000279ms I0326 23:23:08.115602 3952 cni.go:403] [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default] ADD starting CNI request [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default] I0326 23:23:08.124574 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: cec9b93c5b71c46, network: default, NAD default, SandboxID: "cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff", PCI device ID: , UID: "c9f14a57-24f6-4e1a-b18d-376530ba5e6c", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0326 23:23:08.170393 3952 cni.go:424] [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default] ADD finished CNI request [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:5b:11:d5:d9:00\",\"name\":\"cec9b93c5b71c46\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bfb6fda-e971-4907-85cc-715930ca412b\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:23:08.606887 3952 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-ndwt7-ftjr9] creating logical port tsf-tas_segment-backup-installation-ndwt7-ftjr9 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:08.607018 3952 kube.go:256] Updating pod tsf-tas/segment-backup-installation-ndwt7-ftjr9 I0326 23:23:08.621732 3952 pod.go:62] [tsf-tas/segment-backup-installation-ndwt7-ftjr9] pod update took 14.740649ms I0326 23:23:08.621755 3952 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-ndwt7-ftjr9] addLogicalPort annotation time took 14.768834ms I0326 23:23:08.622991 3952 pods.go:271] [tsf-tas/segment-backup-installation-ndwt7-ftjr9] addLogicalPort took 16.126774ms, libovsdb time 840.653µs W0326 23:23:08.918717 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:23:08.918742 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:23:08.918747 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0326 23:23:08.969871 3952 cni.go:403] [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default] I0326 23:23:08.979105 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-ndwt7-ftjr9, hostIfaceName: dfadc46dc3c8235, network: default, NAD default, SandboxID: "dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e", PCI device ID: , UID: "dccd5dba-6ecb-4a8e-82d0-a8d973d4310c", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0326 23:23:09.024014 3952 cni.go:424] [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:42:d1:8a:5c:6a\",\"name\":\"dfadc46dc3c8235\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf852182-9e49-465a-959e-ea408dcc0a17\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:23:09.543708 3952 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-jbrcj-z6q4l] creating logical port tsf-tas_ctlog-createtree-job-jbrcj-z6q4l for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:09.543872 3952 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-jbrcj-z6q4l I0326 23:23:09.557877 3952 pod.go:62] [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l] pod update took 14.049347ms I0326 23:23:09.557903 3952 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-jbrcj-z6q4l] addLogicalPort annotation time took 14.08087ms I0326 23:23:09.559119 3952 pods.go:271] [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l] addLogicalPort took 15.427767ms, libovsdb time 851.856µs I0326 23:23:09.637818 3952 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-68f6dc8b6b-bw49f] creating logical port tsf-tas_fulcio-server-68f6dc8b6b-bw49f for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:09.637914 3952 kube.go:256] Updating pod tsf-tas/fulcio-server-68f6dc8b6b-bw49f I0326 23:23:09.659535 3952 pod.go:62] [tsf-tas/fulcio-server-68f6dc8b6b-bw49f] pod update took 21.645271ms I0326 23:23:09.659559 3952 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-68f6dc8b6b-bw49f] addLogicalPort annotation time took 21.674401ms I0326 23:23:09.660650 3952 pods.go:271] [tsf-tas/fulcio-server-68f6dc8b6b-bw49f] addLogicalPort took 22.876058ms, libovsdb time 718.346µs I0326 23:23:09.714637 3952 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-78855f9d8b-lx4xj] creating logical port tsf-tas_trillian-logserver-78855f9d8b-lx4xj for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:09.714754 3952 kube.go:256] Updating pod tsf-tas/trillian-logserver-78855f9d8b-lx4xj I0326 23:23:09.735098 3952 pod.go:62] [tsf-tas/trillian-logserver-78855f9d8b-lx4xj] pod update took 20.37444ms I0326 23:23:09.735127 3952 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logserver-78855f9d8b-lx4xj] addLogicalPort annotation time took 20.410523ms I0326 23:23:09.736453 3952 pods.go:271] [tsf-tas/trillian-logserver-78855f9d8b-lx4xj] addLogicalPort took 21.839077ms, libovsdb time 933.625µs I0326 23:23:09.904255 3952 cni.go:403] [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default] I0326 23:23:09.914305 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-jbrcj-z6q4l, hostIfaceName: c48071e84ca39e4, network: default, NAD default, SandboxID: "c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3", PCI device ID: , UID: "90549bba-b45e-48ec-9b63-cb19b0a46a8d", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0326 23:23:09.961203 3952 cni.go:424] [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:ec:70:3a:a0:9b\",\"name\":\"c48071e84ca39e4\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/421cf6b7-07b2-43fa-82d4-3782d58a6262\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:23:09.999286 3952 cni.go:403] [tsf-tas/fulcio-server-68f6dc8b6b-bw49f a1e46f4c695157b74ef26ea072ec945a2170c5a37fe27a389365ce4a65a1eb1b network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-68f6dc8b6b-bw49f a1e46f4c695157b74ef26ea072ec945a2170c5a37fe27a389365ce4a65a1eb1b network default NAD default] I0326 23:23:10.008286 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-68f6dc8b6b-bw49f, hostIfaceName: a1e46f4c695157b, network: default, NAD default, SandboxID: "a1e46f4c695157b74ef26ea072ec945a2170c5a37fe27a389365ce4a65a1eb1b", PCI device ID: , UID: "846f647b-711e-4d0e-aff7-2fa69e0e9413", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0326 23:23:10.061159 3952 cni.go:424] [tsf-tas/fulcio-server-68f6dc8b6b-bw49f a1e46f4c695157b74ef26ea072ec945a2170c5a37fe27a389365ce4a65a1eb1b network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-68f6dc8b6b-bw49f a1e46f4c695157b74ef26ea072ec945a2170c5a37fe27a389365ce4a65a1eb1b network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:08:83:da:dc:cb\",\"name\":\"a1e46f4c695157b\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46cbbbc2-82e0-42c1-8ead-1f5399758a49\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:23:10.675002 3952 cni.go:403] [tsf-tas/trillian-logserver-78855f9d8b-lx4xj 7d2dd79b56fe89fc1f668fa6c1d58bb3da2178d205beee837695314e1971114b network default NAD default] ADD starting CNI request [tsf-tas/trillian-logserver-78855f9d8b-lx4xj 7d2dd79b56fe89fc1f668fa6c1d58bb3da2178d205beee837695314e1971114b network default NAD default] I0326 23:23:10.683888 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-78855f9d8b-lx4xj, hostIfaceName: 7d2dd79b56fe89f, network: default, NAD default, SandboxID: "7d2dd79b56fe89fc1f668fa6c1d58bb3da2178d205beee837695314e1971114b", PCI device ID: , UID: "8ec34ee7-558f-4d4a-9a31-1164e467aece", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0326 23:23:10.735605 3952 cni.go:424] [tsf-tas/trillian-logserver-78855f9d8b-lx4xj 7d2dd79b56fe89fc1f668fa6c1d58bb3da2178d205beee837695314e1971114b network default NAD default] ADD finished CNI request [tsf-tas/trillian-logserver-78855f9d8b-lx4xj 7d2dd79b56fe89fc1f668fa6c1d58bb3da2178d205beee837695314e1971114b network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:a2:06:c6:06:3c\",\"name\":\"7d2dd79b56fe89f\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2fd7142-f35e-404a-9f64-5a46851ab38f\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:23:19.836072 3952 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0326 23:23:22.987618 3952 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:23:22.987638 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:23:23.027238 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:23:23.027264 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:23:23.076054 3952 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:23:26.016886 3952 cni.go:403] [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default] I0326 23:23:26.095238 3952 cni.go:424] [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-ndwt7-ftjr9 dfadc46dc3c8235fb6225abe500a0ae8e311fe6863487bb972f7426108c2a16e network default NAD default], result "{}", err I0326 23:23:26.257809 3952 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ndwt7-ftjr9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:26.257845 3952 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-ndwt7-ftjr9 I0326 23:23:26.259133 3952 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-ndwt7-ftjr9, ips: 10.130.0.33 I0326 23:23:26.259161 3952 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ndwt7-ftjr9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:27.019086 3952 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-cdvkb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:27.019669 3952 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-cdvkb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:23:27.107839 3952 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0326 23:23:27.276858 3952 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ndwt7-ftjr9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:27.276923 3952 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ndwt7-ftjr9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:27.718358 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-qn24x for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:27.718458 3952 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x I0326 23:23:27.736027 3952 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x] pod update took 17.591488ms I0326 23:23:27.736054 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x] addLogicalPort annotation time took 17.621248ms I0326 23:23:27.737286 3952 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x] addLogicalPort took 18.949415ms, libovsdb time 838.872µs I0326 23:23:28.047242 3952 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-cdvkb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:28.047270 3952 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-cdvkb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:28.077644 3952 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x e5b28cec767c3b26e45329b3229cf19db44eee1c2b3f221d7078bb45998561b0 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x e5b28cec767c3b26e45329b3229cf19db44eee1c2b3f221d7078bb45998561b0 network default NAD default] I0326 23:23:28.086439 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-qn24x, hostIfaceName: e5b28cec767c3b2, network: default, NAD default, SandboxID: "e5b28cec767c3b26e45329b3229cf19db44eee1c2b3f221d7078bb45998561b0", PCI device ID: , UID: "59ff86cc-296b-45ad-9325-ff3f539af34e", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0326 23:23:28.131888 3952 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x e5b28cec767c3b26e45329b3229cf19db44eee1c2b3f221d7078bb45998561b0 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-qn24x e5b28cec767c3b26e45329b3229cf19db44eee1c2b3f221d7078bb45998561b0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:72:cc:fb:50:c9\",\"name\":\"e5b28cec767c3b2\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b137d3d-c1d7-4689-b2c0-04f7950d0e7e\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:23:28.473495 3952 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:28.474078 3952 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:23:29.109167 3952 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:29.109196 3952 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:23:29.115448 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:23:29.115476 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0326 23:23:31.039132 3952 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0326 23:23:31.039155 3952 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0326 23:23:31.070264 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0326 23:23:31.070285 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0326 23:23:31.138589 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:23:31.139170 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:23:31.139185 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:23:32.924165 3952 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0326 23:23:34.583081 3952 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:23:45.019579 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-s9wqw for pod on switch ip-10-0-0-90.ec2.internal I0326 23:23:45.019710 3952 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw I0326 23:23:45.034150 3952 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw] pod update took 14.494051ms I0326 23:23:45.034176 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw] addLogicalPort annotation time took 14.523929ms I0326 23:23:45.035680 3952 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw] addLogicalPort took 16.1213ms, libovsdb time 1.006552ms I0326 23:23:46.023446 3952 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw a231ff01d64b8ca1b5dea2d35f61186740127c9aefe6753efea7c3724e286200 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw a231ff01d64b8ca1b5dea2d35f61186740127c9aefe6753efea7c3724e286200 network default NAD default] I0326 23:23:46.032109 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-s9wqw, hostIfaceName: a231ff01d64b8ca, network: default, NAD default, SandboxID: "a231ff01d64b8ca1b5dea2d35f61186740127c9aefe6753efea7c3724e286200", PCI device ID: , UID: "eb613a04-c24f-4dce-9dcb-d6552af69bca", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0326 23:23:46.080129 3952 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw a231ff01d64b8ca1b5dea2d35f61186740127c9aefe6753efea7c3724e286200 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-s9wqw a231ff01d64b8ca1b5dea2d35f61186740127c9aefe6753efea7c3724e286200 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:cf:04:9e:df:46\",\"name\":\"a231ff01d64b8ca\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f630d6c-ef86-4008-b127-8feb07aaec20\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:23:57.127574 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:24:03.241977 3952 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-hkpn9-c8tdh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:03.242620 3952 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-hkpn9-c8tdh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:04.104674 3952 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-hkpn9-c8tdh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:04.104700 3952 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-hkpn9-c8tdh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:05.223372 3952 cni.go:403] [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default] DEL starting CNI request [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default] I0326 23:24:05.300464 3952 cni.go:424] [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default] DEL finished CNI request [tsf-tas/test-tsf-tas cec9b93c5b71c46ff3b3ea7fea0595cff36c55d7ed1c0cc4899ac3f6353936ff network default NAD default], result "{}", err I0326 23:24:05.355547 3952 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:05.355583 3952 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0326 23:24:05.357099 3952 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.130.0.32 I0326 23:24:05.357127 3952 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:05.373622 3952 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:05.373640 3952 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:05.380046 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0326 23:24:05.380072 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0326 23:24:09.084328 3952 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-75b8474584-66jzt] creating logical port tsf-tas_rekor-server-75b8474584-66jzt for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:09.084418 3952 kube.go:256] Updating pod tsf-tas/rekor-server-75b8474584-66jzt I0326 23:24:09.098647 3952 pod.go:62] [tsf-tas/rekor-server-75b8474584-66jzt] pod update took 14.250194ms I0326 23:24:09.098675 3952 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-75b8474584-66jzt] addLogicalPort annotation time took 14.28218ms I0326 23:24:09.099981 3952 pods.go:271] [tsf-tas/rekor-server-75b8474584-66jzt] addLogicalPort took 15.67112ms, libovsdb time 879.135µs W0326 23:24:10.345948 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:24:11.255198 3952 cni.go:403] [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default] I0326 23:24:11.354129 3952 cni.go:424] [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-jbrcj-z6q4l c48071e84ca39e46526e01780d95b4254f75a993f770bedc55962761dc0374b3 network default NAD default], result "{}", err I0326 23:24:11.410568 3952 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-jbrcj-z6q4l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:11.410626 3952 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-jbrcj-z6q4l I0326 23:24:11.412178 3952 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-jbrcj-z6q4l, ips: 10.130.0.34 I0326 23:24:11.412209 3952 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-jbrcj-z6q4l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:12.268081 3952 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-jbrcj-z6q4l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:12.268115 3952 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-jbrcj-z6q4l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:24:14.159359 3952 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:24:14.310591 3952 obj_retry.go:464] Detected object tsf/patch-tekton-config-wtzzr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:14.311358 3952 obj_retry.go:464] Detected object tsf/patch-tekton-config-wtzzr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:15.344466 3952 obj_retry.go:458] Detected object tsf/patch-tekton-config-wtzzr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:15.344491 3952 obj_retry.go:458] Detected object tsf/patch-tekton-config-wtzzr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:15.487278 3952 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-z4g8q] creating logical port tsf_tsf-tekton-configuration-z4g8q for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:15.487442 3952 kube.go:256] Updating pod tsf/tsf-tekton-configuration-z4g8q I0326 23:24:15.500624 3952 pod.go:62] [tsf/tsf-tekton-configuration-z4g8q] pod update took 13.213673ms I0326 23:24:15.500652 3952 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-z4g8q] addLogicalPort annotation time took 13.249398ms I0326 23:24:15.501926 3952 pods.go:271] [tsf/tsf-tekton-configuration-z4g8q] addLogicalPort took 14.701494ms, libovsdb time 878.599µs I0326 23:24:15.848518 3952 cni.go:403] [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default] I0326 23:24:15.857230 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-z4g8q, hostIfaceName: 9d2f4e5f95927ec, network: default, NAD default, SandboxID: "9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65", PCI device ID: , UID: "d6bef65b-2e3c-4f91-9827-3112fc739415", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0326 23:24:15.904031 3952 cni.go:424] [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:bc:4d:85:77:12\",\"name\":\"9d2f4e5f95927ec\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0674e462-e82d-47a3-9786-c2344f2aa473\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:24:18.188997 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0326 23:24:19.955114 3952 cni.go:403] [tsf-tas/rekor-server-75b8474584-66jzt 818d81b5536a11d84676dff354832b5e31655cc3293bc92c817be583a40edf90 network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-75b8474584-66jzt 818d81b5536a11d84676dff354832b5e31655cc3293bc92c817be583a40edf90 network default NAD default] I0326 23:24:19.963841 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-75b8474584-66jzt, hostIfaceName: 818d81b5536a11d, network: default, NAD default, SandboxID: "818d81b5536a11d84676dff354832b5e31655cc3293bc92c817be583a40edf90", PCI device ID: , UID: "ca6593cd-bb73-4067-bcd9-defc908e8d97", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0326 23:24:20.011754 3952 cni.go:424] [tsf-tas/rekor-server-75b8474584-66jzt 818d81b5536a11d84676dff354832b5e31655cc3293bc92c817be583a40edf90 network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-75b8474584-66jzt 818d81b5536a11d84676dff354832b5e31655cc3293bc92c817be583a40edf90 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:3d:88:26:a1:77\",\"name\":\"818d81b5536a11d\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c7a1dd4-1bef-4f67-94be-bfa4d34d5b12\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:21.317157 3952 cni.go:403] [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default] I0326 23:24:21.400277 3952 cni.go:424] [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-z4g8q 9d2f4e5f95927ec4f339ca0799a1d95dae01f21927e5cbd8a24a77d8fd912c65 network default NAD default], result "{}", err I0326 23:24:21.456155 3952 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-z4g8q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:21.456189 3952 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-z4g8q I0326 23:24:21.457583 3952 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-z4g8q, ips: 10.130.0.40 I0326 23:24:21.457618 3952 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-z4g8q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:24:22.317033 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:24:22.479539 3952 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-z4g8q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:22.479565 3952 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-z4g8q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:22.520597 3952 obj_retry.go:458] Detected object tsf/patch-tekton-config-wtzzr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:22.520627 3952 obj_retry.go:458] Detected object tsf/patch-tekton-config-wtzzr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:22.528311 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-wtzzr I0326 23:24:22.528339 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-wtzzr I0326 23:24:22.628838 3952 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:22.628927 3952 kube.go:256] Updating pod tsf/test-tsf-pipelines I0326 23:24:22.640921 3952 pod.go:62] [tsf/test-tsf-pipelines] pod update took 12.01563ms I0326 23:24:22.640945 3952 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 12.04552ms I0326 23:24:22.642336 3952 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 13.51685ms, libovsdb time 923.659µs I0326 23:24:22.990033 3952 cni.go:403] [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default] I0326 23:24:22.998661 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: 67704f41028c9a4, network: default, NAD default, SandboxID: "67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0", PCI device ID: , UID: "1b84ac9c-e4a3-4c86-8fe6-af91e16f6270", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0326 23:24:23.044229 3952 cni.go:424] [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:88:94:6b:ea:be\",\"name\":\"67704f41028c9a4\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ff2246eb-474d-4d57-b0c2-3872e9454cda\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:32.382582 3952 cni.go:403] [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default] I0326 23:24:32.466296 3952 cni.go:424] [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines 67704f41028c9a4f717c96bde50fca8e4f02df0b12e092b97942b2fdbe12d8c0 network default NAD default], result "{}", err I0326 23:24:32.516837 3952 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:32.516873 3952 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0326 23:24:32.518321 3952 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.130.0.41 I0326 23:24:32.518349 3952 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:32.533012 3952 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:32.533035 3952 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:32.538381 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0326 23:24:32.538408 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0326 23:24:33.055425 3952 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:33.055528 3952 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0326 23:24:33.068703 3952 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 13.194264ms I0326 23:24:33.068733 3952 base_network_controller_pods.go:937] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 13.232027ms I0326 23:24:33.070053 3952 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 14.647466ms, libovsdb time 953.837µs W0326 23:24:33.328177 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:24:33.417105 3952 cni.go:403] [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default] I0326 23:24:33.425843 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: eb93f639b63efaf, network: default, NAD default, SandboxID: "eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb", PCI device ID: , UID: "588ddc56-adba-4c16-881d-522f62e54595", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0326 23:24:33.471976 3952 cni.go:424] [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:d2:4b:69:88:7c\",\"name\":\"eb93f639b63efaf\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/afb81c6a-e1b4-4e02-8302-44c59ce42f16\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:36.407571 3952 cni.go:403] [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default] I0326 23:24:36.482101 3952 cni.go:424] [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa eb93f639b63efafb4085e4f10ac699ef02accfbc6f2684e813a7a01becc490cb network default NAD default], result "{}", err I0326 23:24:36.536250 3952 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:36.536308 3952 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0326 23:24:36.537624 3952 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.130.0.42 I0326 23:24:36.537656 3952 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:36.556259 3952 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:36.556285 3952 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:36.561796 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0326 23:24:36.561821 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0326 23:24:36.780766 3952 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:36.780891 3952 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0326 23:24:36.794233 3952 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.363657ms I0326 23:24:36.794258 3952 base_network_controller_pods.go:937] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.393975ms I0326 23:24:36.795480 3952 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 14.728765ms, libovsdb time 819.435µs I0326 23:24:37.139793 3952 cni.go:403] [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default] ADD starting CNI request [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default] I0326 23:24:37.148425 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 9e0698dca163fb0, network: default, NAD default, SandboxID: "9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f", PCI device ID: , UID: "0c093323-3145-4b72-b9cd-e4c6f718cbbb", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0326 23:24:37.195726 3952 cni.go:424] [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:99:87:db:7b:63\",\"name\":\"9e0698dca163fb0\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26e7ef8f-4dd4-456c-a08a-95839cf34a65\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:37.458479 3952 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-mjzls-fwm2h] creating logical port tsf-tas_tuf-repository-init-mjzls-fwm2h for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:37.458575 3952 kube.go:256] Updating pod tsf-tas/tuf-repository-init-mjzls-fwm2h I0326 23:24:37.471425 3952 pod.go:62] [tsf-tas/tuf-repository-init-mjzls-fwm2h] pod update took 12.877017ms I0326 23:24:37.471450 3952 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-mjzls-fwm2h] addLogicalPort annotation time took 12.907066ms I0326 23:24:37.472795 3952 pods.go:271] [tsf-tas/tuf-repository-init-mjzls-fwm2h] addLogicalPort took 14.336033ms, libovsdb time 800.357µs I0326 23:24:37.914838 3952 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-2sdm8] creating logical port tsf-tpa_create-db-2sdm8 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:37.914950 3952 kube.go:256] Updating pod tsf-tpa/create-db-2sdm8 I0326 23:24:37.927407 3952 pod.go:62] [tsf-tpa/create-db-2sdm8] pod update took 12.477294ms I0326 23:24:37.927427 3952 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-2sdm8] addLogicalPort annotation time took 12.502064ms I0326 23:24:37.928885 3952 pods.go:271] [tsf-tpa/create-db-2sdm8] addLogicalPort took 14.060937ms, libovsdb time 1.02571ms I0326 23:24:38.321747 3952 cni.go:403] [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default] ADD starting CNI request [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default] I0326 23:24:38.331411 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-2sdm8, hostIfaceName: c118907eafff0f9, network: default, NAD default, SandboxID: "c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec", PCI device ID: , UID: "07df3024-a76a-42b3-b7d0-deff0588bac1", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0326 23:24:38.377968 3952 cni.go:424] [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default] ADD finished CNI request [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:d8:c1:fb:8b:e4\",\"name\":\"c118907eafff0f9\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8bc91e8-821e-4445-9dd5-b9214bd33d16\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:40.823793 3952 cni.go:403] [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default] I0326 23:24:40.833379 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-mjzls-fwm2h, hostIfaceName: d9a3c2195049579, network: default, NAD default, SandboxID: "d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5", PCI device ID: , UID: "051b12bb-45d5-432a-a33f-3e88a967e28c", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0326 23:24:40.883729 3952 cni.go:424] [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:76:f4:3b:08:28\",\"name\":\"d9a3c2195049579\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65b1eb15-e1cc-4520-9280-728c9c1fdf7a\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:43.460599 3952 cni.go:403] [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default] DEL starting CNI request [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default] I0326 23:24:43.555723 3952 cni.go:424] [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default] DEL finished CNI request [tsf-tpa/create-db-2sdm8 c118907eafff0f9a01b75fe5b33cb9f80f9a6d48bacee9db513569140a2297ec network default NAD default], result "{}", err I0326 23:24:43.612539 3952 obj_retry.go:464] Detected object tsf-tpa/create-db-2sdm8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:43.612584 3952 pods.go:176] Deleting pod: tsf-tpa/create-db-2sdm8 I0326 23:24:43.613946 3952 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-2sdm8, ips: 10.130.0.45 I0326 23:24:43.613982 3952 obj_retry.go:464] Detected object tsf-tpa/create-db-2sdm8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:44.635866 3952 obj_retry.go:458] Detected object tsf-tpa/create-db-2sdm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:44.635899 3952 obj_retry.go:458] Detected object tsf-tpa/create-db-2sdm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:44.757890 3952 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-bkx8g] creating logical port tsf-tpa_migrate-db-bkx8g for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:44.757990 3952 kube.go:256] Updating pod tsf-tpa/migrate-db-bkx8g I0326 23:24:44.769342 3952 pod.go:62] [tsf-tpa/migrate-db-bkx8g] pod update took 11.378687ms I0326 23:24:44.769369 3952 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-bkx8g] addLogicalPort annotation time took 11.409019ms I0326 23:24:44.770740 3952 pods.go:271] [tsf-tpa/migrate-db-bkx8g] addLogicalPort took 12.86651ms, libovsdb time 838.205µs I0326 23:24:45.123849 3952 cni.go:403] [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default] I0326 23:24:45.134227 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-bkx8g, hostIfaceName: 0e3973bde5fa8c2, network: default, NAD default, SandboxID: "0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f", PCI device ID: , UID: "f6381eff-caa6-4d3f-bcca-d4e8aad104a7", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0326 23:24:45.180888 3952 cni.go:424] [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:0a:9b:74:29:c4\",\"name\":\"0e3973bde5fa8c2\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e73af1a5-58be-4844-9132-18c093c3bfe4\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:45.477573 3952 cni.go:403] [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default] I0326 23:24:45.550263 3952 cni.go:424] [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-mjzls-fwm2h d9a3c2195049579fa37d17d377e0df5e180b440bf8507d8adad492a86ffee7a5 network default NAD default], result "{}", err I0326 23:24:45.581924 3952 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-mjzls-fwm2h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:45.581957 3952 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-mjzls-fwm2h I0326 23:24:45.583353 3952 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-mjzls-fwm2h, ips: 10.130.0.44 I0326 23:24:45.583384 3952 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-mjzls-fwm2h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:46.483393 3952 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-mjzls-fwm2h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:46.483418 3952 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-mjzls-fwm2h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:47.491493 3952 cni.go:403] [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default] I0326 23:24:47.594461 3952 cni.go:424] [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-bkx8g 0e3973bde5fa8c232b78f1f703bb13931473a8694b4b9b0e3796a0f701e5d57f network default NAD default], result "{}", err I0326 23:24:47.643853 3952 obj_retry.go:464] Detected object tsf-tpa/migrate-db-bkx8g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:47.643891 3952 pods.go:176] Deleting pod: tsf-tpa/migrate-db-bkx8g I0326 23:24:47.645221 3952 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-bkx8g, ips: 10.130.0.46 I0326 23:24:47.645254 3952 obj_retry.go:464] Detected object tsf-tpa/migrate-db-bkx8g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:47.782373 3952 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bkx8g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:47.782401 3952 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bkx8g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:49.523998 3952 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-gbbrh] creating logical port tsf-tas_tuf-bc5768c88-gbbrh for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:49.524094 3952 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-gbbrh I0326 23:24:49.539022 3952 pod.go:62] [tsf-tas/tuf-bc5768c88-gbbrh] pod update took 14.951671ms I0326 23:24:49.539053 3952 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-gbbrh] addLogicalPort annotation time took 14.987114ms I0326 23:24:49.540415 3952 pods.go:271] [tsf-tas/tuf-bc5768c88-gbbrh] addLogicalPort took 16.431479ms, libovsdb time 997.393µs I0326 23:24:50.784851 3952 cni.go:403] [tsf-tas/tuf-bc5768c88-gbbrh 46e71459c8565cb66c19504584017e6fd7972f3dc776bff0dca8026cebbb7ec1 network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-gbbrh 46e71459c8565cb66c19504584017e6fd7972f3dc776bff0dca8026cebbb7ec1 network default NAD default] I0326 23:24:50.793891 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-gbbrh, hostIfaceName: 46e71459c8565cb, network: default, NAD default, SandboxID: "46e71459c8565cb66c19504584017e6fd7972f3dc776bff0dca8026cebbb7ec1", PCI device ID: , UID: "48686a03-2af1-4447-a258-9b7972ca3090", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0326 23:24:50.841538 3952 cni.go:424] [tsf-tas/tuf-bc5768c88-gbbrh 46e71459c8565cb66c19504584017e6fd7972f3dc776bff0dca8026cebbb7ec1 network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-gbbrh 46e71459c8565cb66c19504584017e6fd7972f3dc776bff0dca8026cebbb7ec1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:54:d9:6a:bb:d6\",\"name\":\"46e71459c8565cb\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da46ff00-fc51-4fdf-a600-50b6e459c14c\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:24:50.895452 3952 base_network_controller_pods.go:487] [default/tsf-tpa/server-9b4989678-8ls5q] creating logical port tsf-tpa_server-9b4989678-8ls5q for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:50.895656 3952 kube.go:256] Updating pod tsf-tpa/server-9b4989678-8ls5q I0326 23:24:50.908765 3952 pod.go:62] [tsf-tpa/server-9b4989678-8ls5q] pod update took 13.154612ms I0326 23:24:50.908816 3952 base_network_controller_pods.go:937] [default/tsf-tpa/server-9b4989678-8ls5q] addLogicalPort annotation time took 13.213099ms I0326 23:24:50.910285 3952 pods.go:271] [tsf-tpa/server-9b4989678-8ls5q] addLogicalPort took 14.858788ms, libovsdb time 958.097µs I0326 23:24:50.945235 3952 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-6zfrt] creating logical port tsf-tpa_importer-74cd48f5c7-6zfrt for pod on switch ip-10-0-0-90.ec2.internal I0326 23:24:50.945341 3952 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-6zfrt I0326 23:24:50.960135 3952 pod.go:62] [tsf-tpa/importer-74cd48f5c7-6zfrt] pod update took 14.811384ms I0326 23:24:50.960162 3952 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-6zfrt] addLogicalPort annotation time took 14.845831ms I0326 23:24:50.961505 3952 pods.go:271] [tsf-tpa/importer-74cd48f5c7-6zfrt] addLogicalPort took 16.28503ms, libovsdb time 963.889µs W0326 23:24:53.517007 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0326 23:24:53.534808 3952 obj_retry.go:464] Detected object tsf-tpa/create-importers-bdfhh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:53.535361 3952 obj_retry.go:464] Detected object tsf-tpa/create-importers-bdfhh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:24:54.556707 3952 obj_retry.go:458] Detected object tsf-tpa/create-importers-bdfhh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:24:54.556730 3952 obj_retry.go:458] Detected object tsf-tpa/create-importers-bdfhh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:25:02.270706 3952 cni.go:403] [tsf-tpa/server-9b4989678-8ls5q c2e0109c5cdd7dc8f030da08ded98c238f89a1873aa1dcbcbf7fe8dfa6ab8fcd network default NAD default] ADD starting CNI request [tsf-tpa/server-9b4989678-8ls5q c2e0109c5cdd7dc8f030da08ded98c238f89a1873aa1dcbcbf7fe8dfa6ab8fcd network default NAD default] I0326 23:25:02.279623 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-9b4989678-8ls5q, hostIfaceName: c2e0109c5cdd7dc, network: default, NAD default, SandboxID: "c2e0109c5cdd7dc8f030da08ded98c238f89a1873aa1dcbcbf7fe8dfa6ab8fcd", PCI device ID: , UID: "d7fd85cf-a5f3-4377-83b8-081b11921dd9", MAC: 0a:58:0a:82:00:30, IPs: [10.130.0.48/23] I0326 23:25:02.326731 3952 cni.go:424] [tsf-tpa/server-9b4989678-8ls5q c2e0109c5cdd7dc8f030da08ded98c238f89a1873aa1dcbcbf7fe8dfa6ab8fcd network default NAD default] ADD finished CNI request [tsf-tpa/server-9b4989678-8ls5q c2e0109c5cdd7dc8f030da08ded98c238f89a1873aa1dcbcbf7fe8dfa6ab8fcd network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:50:f3:ef:6d:c4\",\"name\":\"c2e0109c5cdd7dc\"},{\"mac\":\"0a:58:0a:82:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/adf87109-56d9-4781-8675-622ce4f012c8\"}],\"ips\":[{\"address\":\"10.130.0.48/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:25:02.435820 3952 cni.go:403] [tsf-tpa/importer-74cd48f5c7-6zfrt fa606bf1e904f0b1edb885004e4aae1f4a45b70e34ef14b9705b82e656fddfd1 network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-6zfrt fa606bf1e904f0b1edb885004e4aae1f4a45b70e34ef14b9705b82e656fddfd1 network default NAD default] I0326 23:25:02.447305 3952 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-6zfrt, hostIfaceName: fa606bf1e904f0b, network: default, NAD default, SandboxID: "fa606bf1e904f0b1edb885004e4aae1f4a45b70e34ef14b9705b82e656fddfd1", PCI device ID: , UID: "da0ce279-a0fa-4942-bbee-5599e30bd0cc", MAC: 0a:58:0a:82:00:31, IPs: [10.130.0.49/23] I0326 23:25:02.493337 3952 cni.go:424] [tsf-tpa/importer-74cd48f5c7-6zfrt fa606bf1e904f0b1edb885004e4aae1f4a45b70e34ef14b9705b82e656fddfd1 network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-6zfrt fa606bf1e904f0b1edb885004e4aae1f4a45b70e34ef14b9705b82e656fddfd1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:57:7d:d1:1f:7f\",\"name\":\"fa606bf1e904f0b\"},{\"mac\":\"0a:58:0a:82:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4357499-8916-4d7b-b4bd-3e2544fc9fbb\"}],\"ips\":[{\"address\":\"10.130.0.49/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0326 23:25:02.582017 3952 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0326 23:25:04.533553 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0326 23:25:10.025740 3952 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 W0326 23:25:13.584658 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0326 23:25:18.824150 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0326 23:25:26.577278 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=97 I0326 23:25:30.726975 3952 cni.go:403] [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default] DEL starting CNI request [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default] I0326 23:25:30.808627 3952 cni.go:424] [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 9e0698dca163fb028b919bfdd56fdfec79e7fe00be412c0970b5bcf8c996d98f network default NAD default], result "{}", err I0326 23:25:30.865847 3952 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:25:30.865876 3952 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0326 23:25:30.867317 3952 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.130.0.43 I0326 23:25:30.867346 3952 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:25:30.908014 3952 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:25:30.908040 3952 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:25:30.916240 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0326 23:25:30.916271 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0326 23:25:31.548276 3952 namespace.go:98] [image-controller] adding namespace I0326 23:25:31.549228 3952 namespace.go:102] [image-controller] adding namespace took 932.719µs I0326 23:25:31.553451 3952 namespace.go:98] [build-service] adding namespace I0326 23:25:31.553497 3952 namespace.go:98] [integration-service] adding namespace I0326 23:25:31.554044 3952 namespace.go:102] [build-service] adding namespace took 574.642µs I0326 23:25:31.554699 3952 namespace.go:102] [integration-service] adding namespace took 1.187452ms I0326 23:25:31.585815 3952 namespace.go:142] [image-controller] updating namespace I0326 23:25:31.612280 3952 namespace.go:142] [image-controller] updating namespace I0326 23:25:31.626939 3952 namespace.go:142] [build-service] updating namespace I0326 23:25:31.656834 3952 namespace.go:142] [build-service] updating namespace I0326 23:25:31.666147 3952 namespace.go:142] [integration-service] updating namespace I0326 23:25:31.679832 3952 namespace.go:142] [integration-service] updating namespace I0326 23:25:31.746648 3952 namespace.go:142] [build-service] updating namespace I0326 23:25:31.774617 3952 namespace.go:142] [integration-service] updating namespace I0326 23:25:31.805355 3952 namespace.go:98] [release-service] adding namespace I0326 23:25:31.806242 3952 namespace.go:102] [release-service] adding namespace took 871.437µs I0326 23:25:31.838924 3952 namespace.go:142] [release-service] updating namespace I0326 23:25:31.857671 3952 namespace.go:142] [release-service] updating namespace I0326 23:25:31.876693 3952 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-5phcx] creating logical port konflux-ui_tsf-konflux-public-key-5phcx for pod on switch ip-10-0-0-90.ec2.internal I0326 23:25:31.876791 3952 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-5phcx I0326 23:25:31.885786 3952 namespace.go:142] [konflux-ui] updating namespace I0326 23:25:31.904304 3952 pod.go:62] [konflux-ui/tsf-konflux-public-key-5phcx] pod update took 27.553099ms I0326 23:25:31.904328 3952 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-5phcx] addLogicalPort annotation time took 27.580423ms I0326 23:25:31.905503 3952 pods.go:271] [konflux-ui/tsf-konflux-public-key-5phcx] addLogicalPort took 28.822056ms, libovsdb time 797.37µs I0326 23:25:31.934618 3952 namespace.go:98] [konflux-info] adding namespace I0326 23:25:31.935255 3952 namespace.go:102] [konflux-info] adding namespace took 619.975µs I0326 23:25:31.988879 3952 namespace.go:142] [konflux-info] updating namespace I0326 23:25:31.988915 3952 namespace.go:98] [namespace-lister] adding namespace I0326 23:25:31.989588 3952 namespace.go:102] [namespace-lister] adding namespace took 659.499µs I0326 23:25:32.113637 3952 namespace.go:142] [konflux-info] updating namespace I0326 23:25:32.113718 3952 namespace.go:98] [enterprise-contract-service] adding namespace I0326 23:25:32.113871 3952 namespace.go:142] [namespace-lister] updating namespace I0326 23:25:32.114525 3952 namespace.go:102] [enterprise-contract-service] adding namespace took 797.59µs I0326 23:25:32.162848 3952 namespace.go:142] [namespace-lister] updating namespace I0326 23:25:32.265678 3952 namespace.go:142] [image-controller] updating namespace I0326 23:25:32.265847 3952 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:25:32.309862 3952 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:25:32.536605 3952 cni.go:403] [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default] I0326 23:25:32.545292 3952 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-5phcx, hostIfaceName: ee1bdfd5252d52d, network: default, NAD default, SandboxID: "ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa", PCI device ID: , UID: "211fa693-2702-4a6d-966f-791e73886512", MAC: 0a:58:0a:82:00:32, IPs: [10.130.0.50/23] I0326 23:25:32.592432 3952 cni.go:424] [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:a9:e6:d4:57:a1\",\"name\":\"ee1bdfd5252d52d\"},{\"mac\":\"0a:58:0a:82:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3fe05a22-043a-4157-a63e-1ea88f1e4721\"}],\"ips\":[{\"address\":\"10.130.0.50/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:25:32.631842 3952 namespace.go:98] [default-tenant] adding namespace I0326 23:25:32.632601 3952 namespace.go:102] [default-tenant] adding namespace took 735.799µs I0326 23:25:32.714786 3952 namespace.go:142] [default-tenant] updating namespace I0326 23:25:32.744215 3952 namespace.go:142] [default-tenant] updating namespace I0326 23:25:32.818278 3952 namespace.go:98] [segment-bridge] adding namespace I0326 23:25:32.819185 3952 namespace.go:102] [segment-bridge] adding namespace took 888.431µs I0326 23:25:32.959904 3952 namespace.go:142] [segment-bridge] updating namespace I0326 23:25:33.007838 3952 namespace.go:142] [segment-bridge] updating namespace I0326 23:25:33.442602 3952 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0326 23:25:33.443179 3952 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0326 23:25:33.443201 3952 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0326 23:25:33.445717 3952 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0326 23:25:33.518464 3952 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0326 23:25:33.518491 3952 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0326 23:25:33.519327 3952 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0326 23:25:33.902766 3952 namespace.go:142] [build-service] updating namespace I0326 23:25:34.641936 3952 namespace.go:142] [default-tenant] updating namespace I0326 23:25:34.754566 3952 cni.go:403] [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default] I0326 23:25:34.832302 3952 cni.go:424] [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-5phcx ee1bdfd5252d52d656752124882a8deffe65378cba1abbe0aafee98dd3a31aaa network default NAD default], result "{}", err I0326 23:25:35.001574 3952 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-5phcx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:25:35.001616 3952 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-5phcx I0326 23:25:35.002889 3952 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-5phcx, ips: 10.130.0.50 I0326 23:25:35.002916 3952 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-5phcx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:25:35.115709 3952 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:25:35.378704 3952 namespace.go:142] [image-controller] updating namespace I0326 23:25:35.558650 3952 namespace.go:142] [integration-service] updating namespace I0326 23:25:35.808980 3952 namespace.go:142] [konflux-info] updating namespace I0326 23:25:36.040524 3952 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-5phcx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:25:36.040545 3952 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-5phcx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:25:36.102057 3952 namespace.go:142] [namespace-lister] updating namespace I0326 23:25:36.355846 3952 namespace.go:142] [release-service] updating namespace I0326 23:25:36.564115 3952 namespace.go:142] [segment-bridge] updating namespace I0326 23:25:36.685317 3952 namespace.go:142] [build-service] updating namespace I0326 23:25:36.702919 3952 namespace.go:142] [default-tenant] updating namespace I0326 23:25:36.723744 3952 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:25:36.739825 3952 namespace.go:142] [image-controller] updating namespace I0326 23:25:36.753834 3952 namespace.go:142] [integration-service] updating namespace I0326 23:25:36.767368 3952 namespace.go:142] [konflux-info] updating namespace I0326 23:25:36.782671 3952 namespace.go:142] [namespace-lister] updating namespace I0326 23:25:36.792628 3952 namespace.go:142] [release-service] updating namespace I0326 23:25:36.808713 3952 namespace.go:142] [segment-bridge] updating namespace I0326 23:25:36.848105 3952 namespace.go:142] [build-service] updating namespace I0326 23:25:36.936642 3952 namespace.go:142] [default-tenant] updating namespace I0326 23:25:37.032728 3952 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:25:37.131306 3952 namespace.go:142] [image-controller] updating namespace I0326 23:25:37.231649 3952 namespace.go:142] [integration-service] updating namespace I0326 23:25:37.334167 3952 namespace.go:142] [konflux-info] updating namespace I0326 23:25:37.429030 3952 namespace.go:142] [namespace-lister] updating namespace I0326 23:25:37.529328 3952 namespace.go:142] [release-service] updating namespace I0326 23:25:37.633121 3952 namespace.go:142] [segment-bridge] updating namespace W0326 23:25:37.796409 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0326 23:25:38.659324 3952 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0326 23:25:38.681561 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0326 23:25:48.737002 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:25:48.737099 3952 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:25:48.749834 3952 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0326 23:25:48.772836 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:25:48.774632 3952 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:25:48.810137 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:25:48.810154 3952 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:25:49.758566 3952 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:25:49.763521 3952 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:25:51.888958 3952 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:25:51.888978 3952 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0326 23:26:00.097024 3952 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-xkmlm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:00.097604 3952 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-xkmlm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:00.828315 3952 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-xkmlm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:00.828344 3952 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-xkmlm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:00.833973 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-xkmlm I0326 23:26:00.834005 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-xkmlm I0326 23:26:12.601660 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0326 23:26:15.572992 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=648 I0326 23:26:59.338138 3952 obj_retry.go:464] Detected object konflux-ui/proxy-5bc79477fd-2p45s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:59.338761 3952 obj_retry.go:464] Detected object konflux-ui/proxy-5bc79477fd-2p45s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:00.197418 3952 obj_retry.go:458] Detected object konflux-ui/proxy-5bc79477fd-2p45s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:00.197444 3952 obj_retry.go:458] Detected object konflux-ui/proxy-5bc79477fd-2p45s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:00.204038 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5bc79477fd-2p45s I0326 23:27:00.204065 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5bc79477fd-2p45s I0326 23:27:09.452739 3952 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:09.453398 3952 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:09.469757 3952 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:09.469798 3952 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:09.475727 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0326 23:27:09.475748 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0326 23:27:45.365353 3952 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-lfjd4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:45.366045 3952 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-lfjd4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:45.500241 3952 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-lfjd4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:45.500263 3952 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-lfjd4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:45.506152 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-lfjd4 I0326 23:27:45.506174 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-lfjd4 I0326 23:27:59.823494 3952 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kt5xf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:59.824159 3952 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kt5xf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:00.574690 3952 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kt5xf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:00.574719 3952 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kt5xf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:00.581811 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-kt5xf I0326 23:28:00.581835 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-kt5xf I0326 23:28:02.599887 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:28:04.807238 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0326 23:28:08.922641 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0326 23:28:12.015531 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 I0326 23:29:07.586198 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0326 23:29:19.889692 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0326 23:29:28.577408 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=81 I0326 23:29:30.031426 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0326 23:29:37.575469 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=72 I0326 23:29:47.031147 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0326 23:29:47.574659 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=141 I0326 23:30:04.371112 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-c745z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:04.371745 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-c745z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:05.239469 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-c745z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:05.239520 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-c745z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:05.575900 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=44 I0326 23:32:54.826556 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0326 23:33:06.604268 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0326 23:33:24.017598 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0326 23:33:28.579706 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=67 I0326 23:34:01.809967 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0326 23:34:21.028037 3952 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0326 23:34:44.047266 3952 obj_retry.go:458] Detected object tsf-tpa/create-db-2sdm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:44.047318 3952 obj_retry.go:458] Detected object tsf-tpa/create-db-2sdm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:44.052051 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-2sdm8 I0326 23:34:44.052078 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-2sdm8 I0326 23:34:47.026939 3952 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bkx8g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:47.026987 3952 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bkx8g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:47.030509 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-bkx8g I0326 23:34:47.030535 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-bkx8g I0326 23:34:54.037816 3952 obj_retry.go:458] Detected object tsf-tpa/create-importers-bdfhh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:54.037862 3952 obj_retry.go:458] Detected object tsf-tpa/create-importers-bdfhh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:54.042319 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-bdfhh I0326 23:34:54.042359 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-bdfhh I0326 23:34:55.578274 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=29 I0326 23:35:01.602727 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0326 23:35:01.924705 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:36:47.580640 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0326 23:37:37.577264 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0326 23:37:58.891975 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0326 23:38:03.588798 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0326 23:38:18.034281 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0326 23:38:31.578726 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=14 I0326 23:38:37.019893 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0326 23:39:01.607167 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0326 23:39:26.034428 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0326 23:39:48.582087 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=6 I0326 23:40:04.450737 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-n45hw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:04.451413 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-n45hw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:05.323609 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-n45hw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:05.323652 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-n45hw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:05.353508 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:05.353547 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:05.356567 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 I0326 23:40:05.356600 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576110-v8vf4 I0326 23:41:09.454877 3952 namespace.go:98] [default-tenant-managed] adding namespace I0326 23:41:09.455887 3952 namespace.go:102] [default-tenant-managed] adding namespace took 980.032µs I0326 23:41:09.475016 3952 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:41:09.491600 3952 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:41:10.577479 3952 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:41:10.676739 3952 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:41:10.769684 3952 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:41:23.605627 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0326 23:41:28.829148 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0326 23:41:59.511905 3952 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-fpjh7 for pod on switch ip-10-0-0-90.ec2.internal I0326 23:41:59.512048 3952 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 I0326 23:41:59.532720 3952 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7] pod update took 20.702884ms I0326 23:41:59.532752 3952 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7] addLogicalPort annotation time took 20.739589ms I0326 23:41:59.534197 3952 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7] addLogicalPort took 22.306846ms, libovsdb time 868.469µs I0326 23:41:59.872608 3952 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default] I0326 23:41:59.881913 3952 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-fpjh7, hostIfaceName: eb8019be048ba1a, network: default, NAD default, SandboxID: "eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9", PCI device ID: , UID: "b072f92f-d977-48eb-93f5-0495785c1769", MAC: 0a:58:0a:82:00:33, IPs: [10.130.0.51/23] I0326 23:41:59.931102 3952 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:f9:98:c1:55:fe\",\"name\":\"eb8019be048ba1a\"},{\"mac\":\"0a:58:0a:82:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8a09e71e-4782-462b-a3e2-4b2b63c5eb9a\"}],\"ips\":[{\"address\":\"10.130.0.51/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:42:11.726914 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:11.727499 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:42:13.027441 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:13.027472 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:13.027482 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:13.027489 3952 util.go:959] Failed to build node endpoints for node ip-10-0-0-90.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:42:20.812117 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:42:30.928433 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0326 23:42:38.941667 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:38.942409 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:03.460850 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:03.461570 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:48.580748 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=50 I0326 23:43:56.591048 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0326 23:44:00.030291 3952 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0326 23:44:26.582948 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0326 23:44:51.580208 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=14 I0326 23:45:15.584494 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0326 23:45:17.098869 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:17.099517 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:58.036733 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0326 23:45:58.433616 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:58.434227 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:00.318217 3952 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod] creating logical port default-tenant_tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod for pod on switch ip-10-0-0-90.ec2.internal I0326 23:46:00.318347 3952 kube.go:256] Updating pod default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod I0326 23:46:00.345977 3952 pod.go:62] [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod] pod update took 27.670841ms I0326 23:46:00.345999 3952 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod] addLogicalPort annotation time took 27.695948ms I0326 23:46:00.347322 3952 pods.go:271] [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod] addLogicalPort took 29.119973ms, libovsdb time 909.354µs I0326 23:46:00.683781 3952 cni.go:403] [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default] I0326 23:46:00.693431 3952 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod, hostIfaceName: 091c2b8e41c4216, network: default, NAD default, SandboxID: "091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2", PCI device ID: , UID: "bb3f5aa6-74a4-4e75-939e-9e1219c05585", MAC: 0a:58:0a:82:00:34, IPs: [10.130.0.52/23] I0326 23:46:00.747956 3952 cni.go:424] [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:09:bd:22:0f:89\",\"name\":\"091c2b8e41c4216\"},{\"mac\":\"0a:58:0a:82:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/761dd5b5-f103-4425-bab7-e5f8529ab4e6\"}],\"ips\":[{\"address\":\"10.130.0.52/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0326 23:46:18.894562 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0326 23:46:48.021948 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0326 23:46:59.265612 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:59.266266 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:00.966835 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:00.967474 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:02.659492 3952 cni.go:403] [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default] I0326 23:47:02.756447 3952 cni.go:424] [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod 091c2b8e41c421647f83e08aa5649a8d8a05e6875009f8f276c1f07bcb0048e2 network default NAD default], result "{}", err I0326 23:47:02.839883 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:02.839924 3952 pods.go:176] Deleting pod: default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod I0326 23:47:02.844427 3952 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod, ips: 10.130.0.52 I0326 23:47:02.844472 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:02.966304 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:02.966986 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:06.213135 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:06.213815 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:06.237291 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:06.237877 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:09.225973 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:09.226617 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:12.279335 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:12.280005 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:22.340111 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:22.340888 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:36.581199 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=225 I0326 23:47:58.814785 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0326 23:48:01.609693 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0326 23:48:19.876081 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:19.876715 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:49.037876 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0326 23:48:57.608193 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0326 23:49:43.933089 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:50:03.990996 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dckrr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:03.991615 3952 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dckrr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.865615 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dckrr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:04.865648 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dckrr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:04.895010 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:04.895038 3952 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:04.899433 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww I0326 23:50:04.899462 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-ctnww I0326 23:50:09.654263 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:09.654940 3952 obj_retry.go:464] Detected object default-tenant/tsf-comp-irgo-on-push-84cx7-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:14.585713 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=7 I0326 23:50:37.032631 3952 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0326 23:50:57.410068 3952 obj_retry.go:464] Detected object default-tenant/my-integration-test-odtc-5pf5j-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:57.410665 3952 obj_retry.go:464] Detected object default-tenant/my-integration-test-odtc-5pf5j-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:51:14.831411 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0326 23:51:53.583385 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=49 I0326 23:52:06.593450 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0326 23:52:14.039381 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0326 23:52:40.582900 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=10 I0326 23:53:09.584269 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=52 I0326 23:53:30.765044 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-r6f7n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:30.765740 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-r6f7n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:31.144986 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-r6f7n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:53:31.145012 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-r6f7n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:53:31.152066 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-r6f7n I0326 23:53:31.152091 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-r6f7n I0326 23:53:53.897137 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0326 23:54:19.817012 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0326 23:54:35.587433 3952 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=11 I0326 23:55:00.745800 3952 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default] DEL starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default] I0326 23:55:00.825457 3952 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default] DEL finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 eb8019be048ba1af8950e43528dfcdb22ae5e2da05f1c5b35df7bd747a2966a9 network default NAD default], result "{}", err I0326 23:55:00.886818 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-jrzm6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:00.887513 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-jrzm6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:00.908579 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-8gw5k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:00.909182 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-8gw5k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:00.919273 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:00.919371 3952 pods.go:176] Deleting pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 I0326 23:55:00.920953 3952 pods.go:236] Attempting to release IPs for pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7, ips: 10.130.0.51 I0326 23:55:00.920979 3952 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:01.267831 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:55:01.267863 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:55:01.275507 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 I0326 23:55:01.275533 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-fpjh7 I0326 23:55:01.693640 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-jrzm6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:55:01.693690 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-jrzm6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:55:01.699169 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-jrzm6 I0326 23:55:01.699193 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-jrzm6 I0326 23:55:01.713745 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-8gw5k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:55:01.713765 3952 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-8gw5k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:55:01.719315 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-8gw5k I0326 23:55:01.719342 3952 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-8gw5k I0326 23:55:11.024918 3952 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0326 23:55:28.610822 3952 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8