+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-231.ec2.internal ++ [[ -n ip-10-0-0-231.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-231.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0326 23:32:27.658552341 - disable conntrack on geneve port' I0326 23:32:27.658552341 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0326 23:32:27.674657768 - starting ovnkube-node + echo 'I0326 23:32:27.674657768 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-231.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-231.ec2.internal --init-node ip-10-0-0-231.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-231.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0326 23:32:27.713779 3822 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0326 23:32:27.713821 3822 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-8ed5abe7ae.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0326 23:32:27.715399 3822 kube.go:419] Waiting for certificate I0326 23:32:27.715440 3822 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0326 23:32:27.715482 3822 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0326 23:32:27.716530 3822 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:32:27.743728 3822 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:27.743752 3822 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:27.746173 3822 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:27.748800 3822 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-grvhv" I0326 23:32:27.758509 3822 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-grvhv" I0326 23:32:27.758593 3822 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:28.716202 3822 kube.go:426] Certificate found I0326 23:32:28.716700 3822 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:32:28.717835 3822 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0326 23:32:28.762660 3822 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-27 23:27:27 +0000 UTC" deadline="2026-03-27 20:27:17.210776963 +0000 UTC" I0326 23:32:28.762697 3822 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h54m48.448091455s" I0326 23:32:28.779869 3822 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:32:28.794798 3822 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0326 23:32:28.796381 3822 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0326 23:32:28.796520 3822 factory.go:531] Starting watch factory I0326 23:32:28.796646 3822 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.796685 3822 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.797899 3822 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.797931 3822 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.797948 3822 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.797950 3822 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.798131 3822 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.798156 3822 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.798835 3822 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.798899 3822 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.799527 3822 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.799546 3822 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.805786 3822 controller_manager.go:368] Starting the ovnkube controller I0326 23:32:28.805799 3822 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-231.ec2.internal I0326 23:32:28.805852 3822 controller_manager.go:393] NBDB zone sync took: 45.046µs I0326 23:32:28.805867 3822 factory.go:531] Starting watch factory I0326 23:32:28.815713 3822 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.816624 3822 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.817635 3822 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0326 23:32:28.817924 3822 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.819037 3822 ovn_northd.go:121] Registering OVN northd metrics I0326 23:32:28.820448 3822 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.820872 3822 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.821714 3822 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0326 23:32:28.821776 3822 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0326 23:32:28.825695 3822 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:28.837146 3822 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0326 23:32:28.902311 3822 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:32:28.902333 3822 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:32:28.902337 3822 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:32:28.902340 3822 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:32:28.902343 3822 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:32:28.902346 3822 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:32:28.902429 3822 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:28.902445 3822 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:28.902452 3822 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:28.902468 3822 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:28.905972 3822 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:28.905982 3822 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:28.907856 3822 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:32:28.907868 3822 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:32:28.907929 3822 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.907940 3822 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.911244 3822 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:32:28.911255 3822 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:32:28.911237 3822 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.911274 3822 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:32:28.911315 3822 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:32:28.911322 3822 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:32:28.911325 3822 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:32:28.913204 3822 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:32:28.913280 3822 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.913294 3822 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.916093 3822 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.917023 3822 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:32:28.917033 3822 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:32:28.918669 3822 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:32:28.918716 3822 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.918726 3822 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.921523 3822 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.922464 3822 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:32:28.924158 3822 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:32:28.924212 3822 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.924222 3822 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.927158 3822 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.928127 3822 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:32:28.929279 3822 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:32:28.929329 3822 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.929340 3822 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.932182 3822 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.934152 3822 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:32:28.934870 3822 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:32:28.935071 3822 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:32:28.935095 3822 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:32:28.938110 3822 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:32:28.940045 3822 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:32:28.940121 3822 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.940141 3822 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.940157 3822 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.940164 3822 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:32:28.940168 3822 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.943147 3822 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.943348 3822 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:28.945290 3822 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:32:28.945300 3822 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:32:28.945307 3822 factory.go:636] Watch Factory start up complete, took: 148.792147ms I0326 23:32:28.945398 3822 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0326 23:32:28.945469 3822 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:32:28.950889 3822 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:32:28.955924 3822 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:32:28.955934 3822 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:32:28.955941 3822 factory.go:636] Watch Factory start up complete, took: 150.076344ms I0326 23:32:28.955947 3822 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-231.ec2.internal" zone I0326 23:32:29.006753 3822 default_node_network_controller.go:709] Initializing the default node network controller I0326 23:32:29.072278 3822 udn_isolation.go:101] Starting UDN host isolation manager I0326 23:32:29.079204 3822 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0326 23:32:29.157780 3822 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0326 23:32:29.157867 3822 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0326 23:32:29.157887 3822 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0326 23:32:29.188389 3822 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0326 23:32:29.188450 3822 default_node_network_controller.go:808] Node ip-10-0-0-231.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0326 23:32:29.193325 3822 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.231"] k8s.ovn.org/zone-name:ip-10-0-0-231.ec2.internal] on node ip-10-0-0-231.ec2.internal I0326 23:32:29.215067 3822 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0326 23:32:29.220364 3822 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0326 23:32:29.220509 3822 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0326 23:32:29.220619 3822 gateway_init.go:263] Preparing Gateway I0326 23:32:29.220631 3822 gateway_shared_intf.go:1658] Creating new gateway I0326 23:32:29.307838 3822 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:32:29.313565 3822 iptables.go:108] Creating table: filter chain: OUTPUT I0326 23:32:29.319496 3822 gateway_shared_intf.go:1787] Gateway Creation Complete I0326 23:32:29.319511 3822 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-231.ec2.internal","mac-address":"12:f7:2e:b8:9a:8b","ip-addresses":["10.0.0.231/24"],"ip-address":"10.0.0.231/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:c3c005d2-39af-4b39-a3c0-c73a9ecd5b73 k8s.ovn.org/node-encap-ips:["10.0.0.231"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.231/24"} k8s.ovn.org/zone-name:ip-10-0-0-231.ec2.internal] on node ip-10-0-0-231.ec2.internal I0326 23:32:29.341379 3822 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:29.456662 3822 controller_manager.go:427] Waiting for node in zone sync took: 500.700057ms I0326 23:32:29.463357 3822 controller_manager.go:608] Switching to transit router for layer2 networks I0326 23:32:29.463372 3822 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-231.ec2.internal I0326 23:32:29.478801 3822 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:32:29.484699 3822 controller_manager.go:288] SCTP support detected in OVN I0326 23:32:29.485214 3822 services_controller.go:69] Creating services controller for network=default I0326 23:32:29.485322 3822 udn_enabled_svc.go:80] Waiting for service informer to sync I0326 23:32:29.485417 3822 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0326 23:32:29.485456 3822 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0326 23:32:29.485469 3822 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0326 23:32:29.485479 3822 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:32:29.485626 3822 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0326 23:32:29.485671 3822 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0326 23:32:29.485699 3822 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 45.661µs I0326 23:32:29.485715 3822 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 36.282µs I0326 23:32:29.485719 3822 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 48.886µs I0326 23:32:29.485725 3822 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0326 23:32:29.485729 3822 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0326 23:32:29.485737 3822 default_network_controller.go:321] Starting the default network controller I0326 23:32:29.485759 3822 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 9.153µs I0326 23:32:29.485772 3822 udn_enabled_svc.go:95] Performing full resync I0326 23:32:29.486006 3822 udn_enabled_svc.go:99] Waiting for handler to sync I0326 23:32:29.486014 3822 udn_enabled_svc.go:104] Starting worker I0326 23:32:29.493297 3822 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0326 23:32:29.493348 3822 repair.go:33] Syncing exgw routes took 39.186µs I0326 23:32:29.493367 3822 default_network_controller.go:407] Starting all the Watchers... I0326 23:32:29.493486 3822 namespace.go:98] [openshift] adding namespace I0326 23:32:29.493497 3822 namespace.go:98] [openshift-controller-manager] adding namespace I0326 23:32:29.493497 3822 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0326 23:32:29.493486 3822 namespace.go:98] [openshift-backplane-tam] adding namespace I0326 23:32:29.493501 3822 namespace.go:98] [openshift-network-console] adding namespace I0326 23:32:29.493513 3822 namespace.go:98] [dedicated-admin] adding namespace I0326 23:32:29.493523 3822 namespace.go:98] [openshift-config] adding namespace I0326 23:32:29.493524 3822 namespace.go:98] [openshift-insights] adding namespace I0326 23:32:29.493523 3822 namespace.go:98] [openshift-cluster-version] adding namespace I0326 23:32:29.493529 3822 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0326 23:32:29.493531 3822 namespace.go:98] [openshift-customer-monitoring] adding namespace I0326 23:32:29.493536 3822 namespace.go:98] [openshift-marketplace] adding namespace I0326 23:32:29.493533 3822 namespace.go:98] [openshift-logging] adding namespace I0326 23:32:29.493546 3822 namespace.go:98] [openshift-console] adding namespace I0326 23:32:29.493486 3822 namespace.go:98] [openshift-network-node-identity] adding namespace I0326 23:32:29.494008 3822 namespace.go:102] [openshift] adding namespace took 510.781µs I0326 23:32:29.494023 3822 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0326 23:32:29.494456 3822 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 428.441µs I0326 23:32:29.494468 3822 namespace.go:98] [openshift-backplane-mobb] adding namespace I0326 23:32:29.494931 3822 namespace.go:102] [openshift-backplane-mobb] adding namespace took 458.639µs I0326 23:32:29.494942 3822 namespace.go:98] [openshift-backplane-cse] adding namespace I0326 23:32:29.495423 3822 namespace.go:102] [openshift-controller-manager] adding namespace took 1.917607ms I0326 23:32:29.495433 3822 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0326 23:32:29.496007 3822 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 2.497486ms I0326 23:32:29.496020 3822 namespace.go:98] [openshift-ingress] adding namespace I0326 23:32:29.496910 3822 namespace.go:102] [openshift-backplane-tam] adding namespace took 3.396041ms I0326 23:32:29.496931 3822 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0326 23:32:29.497353 3822 namespace.go:102] [openshift-network-console] adding namespace took 3.837987ms I0326 23:32:29.497372 3822 namespace.go:98] [openshift-backplane-csm] adding namespace I0326 23:32:29.497787 3822 namespace.go:102] [dedicated-admin] adding namespace took 4.265893ms I0326 23:32:29.497803 3822 namespace.go:98] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] adding namespace I0326 23:32:29.498198 3822 namespace.go:102] [openshift-config] adding namespace took 4.670132ms I0326 23:32:29.498214 3822 namespace.go:98] [openshift-package-operator] adding namespace I0326 23:32:29.498636 3822 namespace.go:102] [openshift-insights] adding namespace took 5.104616ms I0326 23:32:29.498653 3822 namespace.go:98] [openshift-network-diagnostics] adding namespace I0326 23:32:29.499059 3822 namespace.go:102] [openshift-cluster-version] adding namespace took 5.523607ms I0326 23:32:29.499076 3822 namespace.go:98] [openshift-ingress-canary] adding namespace I0326 23:32:29.499480 3822 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.94207ms I0326 23:32:29.499494 3822 namespace.go:98] [openshift-backplane-cee] adding namespace I0326 23:32:29.500018 3822 namespace.go:102] [openshift-marketplace] adding namespace took 6.475865ms I0326 23:32:29.500029 3822 namespace.go:98] [openshift-operators] adding namespace I0326 23:32:29.500549 3822 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.006799ms I0326 23:32:29.500567 3822 namespace.go:98] [openshift-backplane-srep] adding namespace I0326 23:32:29.500980 3822 namespace.go:102] [openshift-logging] adding namespace took 7.435053ms I0326 23:32:29.500997 3822 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0326 23:32:29.501425 3822 namespace.go:102] [openshift-console] adding namespace took 7.870483ms I0326 23:32:29.501444 3822 namespace.go:98] [openshift-host-network] adding namespace I0326 23:32:29.501831 3822 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.229834ms I0326 23:32:29.501845 3822 namespace.go:98] [openshift-dns] adding namespace I0326 23:32:29.502249 3822 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.301172ms I0326 23:32:29.502277 3822 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0326 23:32:29.502680 3822 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.242056ms I0326 23:32:29.502692 3822 namespace.go:98] [openshift-monitoring] adding namespace I0326 23:32:29.503001 3822 namespace.go:102] [openshift-ingress] adding namespace took 6.977351ms I0326 23:32:29.503011 3822 namespace.go:98] [openshift-multus] adding namespace I0326 23:32:29.503590 3822 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.65319ms I0326 23:32:29.503600 3822 namespace.go:98] [kube-system] adding namespace I0326 23:32:29.504011 3822 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.632766ms I0326 23:32:29.504024 3822 namespace.go:98] [openshift-machine-api] adding namespace I0326 23:32:29.504482 3822 namespace.go:102] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] adding namespace took 6.673518ms I0326 23:32:29.504493 3822 namespace.go:98] [openshift-operators-redhat] adding namespace W0326 23:32:29.504823 3822 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-99h22: pod openshift-network-diagnostics/network-check-target-99h22: no pod IPs found W0326 23:32:29.504835 3822 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-vshpw: pod openshift-network-diagnostics/network-check-target-vshpw: no pod IPs found I0326 23:32:29.504865 3822 namespace.go:102] [openshift-package-operator] adding namespace took 6.647307ms I0326 23:32:29.504874 3822 namespace.go:98] [openshift-dns-operator] adding namespace I0326 23:32:29.505477 3822 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.818495ms I0326 23:32:29.505489 3822 namespace.go:98] [openshift-apiserver] adding namespace I0326 23:32:29.505991 3822 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.907095ms I0326 23:32:29.506009 3822 namespace.go:98] [openshift-route-controller-manager] adding namespace I0326 23:32:29.506426 3822 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.92746ms I0326 23:32:29.506437 3822 namespace.go:98] [openshift-backplane] adding namespace I0326 23:32:29.506888 3822 namespace.go:102] [openshift-operators] adding namespace took 6.854569ms I0326 23:32:29.506897 3822 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0326 23:32:29.507416 3822 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.844647ms I0326 23:32:29.507425 3822 namespace.go:98] [openshift-authentication-operator] adding namespace I0326 23:32:29.507851 3822 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.849066ms I0326 23:32:29.507858 3822 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0326 23:32:29.508421 3822 namespace.go:102] [openshift-host-network] adding namespace took 6.968816ms I0326 23:32:29.508438 3822 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0326 23:32:29.508817 3822 namespace.go:102] [openshift-dns] adding namespace took 6.96693ms I0326 23:32:29.508830 3822 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0326 23:32:29.509321 3822 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.036739ms W0326 23:32:29.509714 3822 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-4wfbf: pod openshift-multus/network-metrics-daemon-4wfbf: no pod IPs found W0326 23:32:29.509723 3822 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-nfmjv: pod openshift-multus/network-metrics-daemon-nfmjv: no pod IPs found I0326 23:32:29.509765 3822 namespace.go:102] [openshift-monitoring] adding namespace took 7.069285ms I0326 23:32:29.509775 3822 namespace.go:98] [openshift-kube-scheduler] adding namespace W0326 23:32:29.510108 3822 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-z9lfw: pod kube-system/global-pull-secret-syncer-z9lfw: no pod IPs found I0326 23:32:29.510157 3822 namespace.go:102] [openshift-multus] adding namespace took 7.142627ms I0326 23:32:29.510165 3822 namespace.go:98] [default] adding namespace I0326 23:32:29.510651 3822 namespace.go:102] [kube-system] adding namespace took 7.046393ms I0326 23:32:29.510662 3822 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0326 23:32:29.511169 3822 namespace.go:102] [openshift-machine-api] adding namespace took 7.138827ms I0326 23:32:29.511187 3822 namespace.go:98] [openshift-node] adding namespace I0326 23:32:29.511594 3822 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.096988ms I0326 23:32:29.511605 3822 namespace.go:98] [openshift-service-ca-operator] adding namespace I0326 23:32:29.512098 3822 namespace.go:102] [openshift-dns-operator] adding namespace took 7.218508ms I0326 23:32:29.512113 3822 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0326 23:32:29.512521 3822 namespace.go:102] [openshift-apiserver] adding namespace took 7.027387ms I0326 23:32:29.512531 3822 namespace.go:98] [openshift-console-operator] adding namespace I0326 23:32:29.513033 3822 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.016915ms I0326 23:32:29.513051 3822 namespace.go:98] [kube-public] adding namespace I0326 23:32:29.513458 3822 namespace.go:102] [openshift-backplane] adding namespace took 7.016773ms I0326 23:32:29.513469 3822 namespace.go:98] [openshift-machine-config-operator] adding namespace I0326 23:32:29.513968 3822 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.064127ms I0326 23:32:29.513985 3822 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0326 23:32:29.514398 3822 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.969061ms I0326 23:32:29.514407 3822 namespace.go:98] [openshift-network-operator] adding namespace I0326 23:32:29.514903 3822 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.039075ms I0326 23:32:29.514914 3822 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0326 23:32:29.515279 3822 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.836434ms I0326 23:32:29.515288 3822 namespace.go:98] [openshift-config-managed] adding namespace I0326 23:32:29.515803 3822 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.966548ms I0326 23:32:29.515821 3822 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0326 23:32:29.516175 3822 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.394844ms I0326 23:32:29.516186 3822 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0326 23:32:29.516644 3822 namespace.go:102] [default] adding namespace took 6.475549ms I0326 23:32:29.516655 3822 namespace.go:98] [openshift-must-gather-operator] adding namespace I0326 23:32:29.517000 3822 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.333452ms I0326 23:32:29.517013 3822 namespace.go:98] [openshift-authentication] adding namespace I0326 23:32:29.517381 3822 namespace.go:102] [openshift-node] adding namespace took 6.187856ms I0326 23:32:29.517391 3822 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0326 23:32:29.518102 3822 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.493561ms I0326 23:32:29.518112 3822 namespace.go:98] [openshift-etcd] adding namespace I0326 23:32:29.518438 3822 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.320498ms I0326 23:32:29.518445 3822 namespace.go:98] [openshift-image-registry] adding namespace I0326 23:32:29.518925 3822 namespace.go:102] [openshift-console-operator] adding namespace took 6.389029ms I0326 23:32:29.518939 3822 namespace.go:98] [openshift-infra] adding namespace I0326 23:32:29.519279 3822 namespace.go:102] [kube-public] adding namespace took 6.223563ms I0326 23:32:29.519288 3822 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0326 23:32:29.519799 3822 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.323949ms I0326 23:32:29.520125 3822 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.134715ms I0326 23:32:29.520524 3822 namespace.go:102] [openshift-network-operator] adding namespace took 6.111587ms I0326 23:32:29.520537 3822 namespace.go:98] [openshift-config-operator] adding namespace I0326 23:32:29.520974 3822 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.053936ms I0326 23:32:29.520991 3822 namespace.go:98] [openshift-kube-apiserver] adding namespace I0326 23:32:29.521418 3822 namespace.go:102] [openshift-config-managed] adding namespace took 6.125976ms I0326 23:32:29.521430 3822 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0326 23:32:29.521776 3822 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.950497ms I0326 23:32:29.521787 3822 namespace.go:98] [openshift-ingress-operator] adding namespace I0326 23:32:29.522201 3822 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.009986ms I0326 23:32:29.522215 3822 namespace.go:98] [openshift-console-user-settings] adding namespace I0326 23:32:29.522576 3822 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.914031ms I0326 23:32:29.523025 3822 namespace.go:102] [openshift-authentication] adding namespace took 6.006858ms I0326 23:32:29.523465 3822 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.068778ms I0326 23:32:29.523478 3822 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0326 23:32:29.523800 3822 namespace.go:102] [openshift-etcd] adding namespace took 5.683996ms I0326 23:32:29.523810 3822 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0326 23:32:29.524253 3822 namespace.go:102] [openshift-image-registry] adding namespace took 5.802544ms I0326 23:32:29.524281 3822 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0326 23:32:29.524596 3822 namespace.go:102] [openshift-infra] adding namespace took 5.651649ms I0326 23:32:29.524607 3822 namespace.go:98] [openshift-apiserver-operator] adding namespace I0326 23:32:29.525099 3822 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.805171ms I0326 23:32:29.525513 3822 namespace.go:102] [openshift-config-operator] adding namespace took 4.971417ms I0326 23:32:29.525925 3822 namespace.go:102] [openshift-kube-apiserver] adding namespace took 4.928295ms I0326 23:32:29.526337 3822 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 4.901826ms I0326 23:32:29.526349 3822 namespace.go:98] [kube-node-lease] adding namespace I0326 23:32:29.526766 3822 namespace.go:102] [openshift-ingress-operator] adding namespace took 4.975169ms I0326 23:32:29.527173 3822 namespace.go:102] [openshift-console-user-settings] adding namespace took 4.953241ms I0326 23:32:29.527620 3822 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 4.138071ms I0326 23:32:29.528011 3822 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 4.197674ms I0326 23:32:29.528453 3822 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 4.165461ms I0326 23:32:29.528819 3822 namespace.go:102] [openshift-apiserver-operator] adding namespace took 4.206287ms I0326 23:32:29.529179 3822 namespace.go:102] [kube-node-lease] adding namespace took 2.825032ms I0326 23:32:29.529557 3822 zone_ic_handler.go:186] Time taken to create transit switch: 276.62µs I0326 23:32:29.529624 3822 master.go:532] Adding or Updating local node "ip-10-0-0-231.ec2.internal" for network "default" I0326 23:32:29.529665 3822 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-151.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-151.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-151.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-151.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-151.ec2.internal W0326 23:32:29.536182 3822 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-99h22: pod openshift-network-diagnostics/network-check-target-99h22: no pod IPs found W0326 23:32:29.536193 3822 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-z9lfw: pod kube-system/global-pull-secret-syncer-z9lfw: no pod IPs found W0326 23:32:29.536210 3822 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-nfmjv: pod openshift-multus/network-metrics-daemon-nfmjv: no pod IPs found I0326 23:32:29.537057 3822 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-231.ec2.internal for the network default I0326 23:32:29.537806 3822 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0326 23:32:29.537835 3822 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0326 23:32:29.537849 3822 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0326 23:32:29.537868 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-231.ec2.internal in network "default" I0326 23:32:29.537914 3822 node_tracker.go:169] Node ip-10-0-0-231.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:29.537924 3822 services_controller.go:574] Full service sync requested for network=default I0326 23:32:29.537933 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-151.ec2.internal in network "default" I0326 23:32:29.537944 3822 node_tracker.go:235] Node ip-10-0-2-151.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-151.ec2.internal" I0326 23:32:29.537952 3822 node_tracker.go:169] Node ip-10-0-2-151.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:29.537956 3822 services_controller.go:574] Full service sync requested for network=default I0326 23:32:29.638077 3822 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0326 23:32:29.638095 3822 services_controller.go:208] Setting up event handlers for services for network=default I0326 23:32:29.638137 3822 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0326 23:32:29.638246 3822 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0326 23:32:29.638256 3822 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0326 23:32:29.738487 3822 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0326 23:32:29.738508 3822 repair.go:60] Starting repairing loop for services I0326 23:32:29.738591 3822 repair.go:131] Deleted 0 stale service LBs I0326 23:32:29.738609 3822 repair.go:137] Deleted 0 stale Chassis Template Vars I0326 23:32:29.738624 3822 repair.go:62] Finished repairing loop for services: 119.505µs I0326 23:32:29.738647 3822 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0326 23:32:29.738657 3822 services_controller.go:254] Starting workers for network=default I0326 23:32:29.738949 3822 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-vshpw, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-vshpw: suppressed error logged: pod openshift-network-diagnostics/network-check-target-vshpw: no pod IPs found I0326 23:32:29.738967 3822 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-99h22] creating logical port openshift-network-diagnostics_network-check-target-99h22 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:29.738986 3822 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-4wfbf, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-4wfbf: suppressed error logged: pod openshift-multus/network-metrics-daemon-4wfbf: no pod IPs found I0326 23:32:29.739174 3822 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-99h22 I0326 23:32:29.739411 3822 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-nfmjv] creating logical port openshift-multus_network-metrics-daemon-nfmjv for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:29.739428 3822 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-z9lfw] creating logical port kube-system_global-pull-secret-syncer-z9lfw for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:29.739502 3822 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-z9lfw I0326 23:32:29.739504 3822 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-nfmjv I0326 23:32:29.762994 3822 pod.go:62] [kube-system/global-pull-secret-syncer-z9lfw] pod update took 23.511805ms I0326 23:32:29.763014 3822 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-z9lfw] addLogicalPort annotation time took 23.533341ms I0326 23:32:29.763479 3822 pod.go:62] [openshift-multus/network-metrics-daemon-nfmjv] pod update took 23.999853ms I0326 23:32:29.763500 3822 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-nfmjv] addLogicalPort annotation time took 24.022422ms I0326 23:32:29.763847 3822 pods.go:271] [kube-system/global-pull-secret-syncer-z9lfw] addLogicalPort took 24.425546ms, libovsdb time 611.883µs I0326 23:32:29.764241 3822 pods.go:271] [openshift-multus/network-metrics-daemon-nfmjv] addLogicalPort took 24.835794ms, libovsdb time 539.329µs I0326 23:32:29.766056 3822 pod.go:62] [openshift-network-diagnostics/network-check-target-99h22] pod update took 27.004105ms I0326 23:32:29.766075 3822 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-99h22] addLogicalPort annotation time took 27.025479ms I0326 23:32:29.766916 3822 pods.go:271] [openshift-network-diagnostics/network-check-target-99h22] addLogicalPort took 27.96001ms, libovsdb time 632.383µs I0326 23:32:29.767314 3822 admin_network_policy_controller.go:227] Starting controller default-network-controller I0326 23:32:29.767334 3822 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0326 23:32:29.767352 3822 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0326 23:32:29.767356 3822 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0326 23:32:29.767533 3822 repair.go:29] Repairing admin network policies took 171.408µs I0326 23:32:29.767628 3822 repair.go:92] Repairing baseline admin network policies took 87µs I0326 23:32:29.767951 3822 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0326 23:32:29.767974 3822 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0326 23:32:29.770408 3822 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0326 23:32:29.770427 3822 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0326 23:32:29.770438 3822 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0326 23:32:29.772298 3822 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0326 23:32:29.772316 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0326 23:32:29.772325 3822 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0326 23:32:29.773634 3822 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0326 23:32:29.773649 3822 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0326 23:32:29.773656 3822 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0326 23:32:29.775518 3822 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0326 23:32:29.775539 3822 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0326 23:32:29.775554 3822 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0326 23:32:29.776619 3822 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0326 23:32:29.776636 3822 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0326 23:32:29.776646 3822 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0326 23:32:29.777709 3822 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0326 23:32:29.777726 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0326 23:32:29.777734 3822 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0326 23:32:29.778140 3822 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0326 23:32:29.778156 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0326 23:32:29.778163 3822 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0326 23:32:29.778764 3822 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0326 23:32:29.778780 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0326 23:32:29.779101 3822 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0326 23:32:29.779113 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:32:29.780736 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0326 23:32:29.780754 3822 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0326 23:32:29.780761 3822 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0326 23:32:29.781071 3822 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0326 23:32:29.781089 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0326 23:32:29.781096 3822 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0326 23:32:29.781438 3822 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0326 23:32:29.781452 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0326 23:32:29.781466 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0326 23:32:29.782008 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:32:29.782023 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0326 23:32:29.782030 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0326 23:32:29.783751 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0326 23:32:29.783773 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0326 23:32:29.783781 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0326 23:32:29.784329 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0326 23:32:29.784346 3822 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0326 23:32:29.784358 3822 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0326 23:32:29.785320 3822 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0326 23:32:29.785335 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0326 23:32:29.785343 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0326 23:32:29.785867 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0326 23:32:29.785885 3822 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0326 23:32:29.785893 3822 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0326 23:32:29.786381 3822 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0326 23:32:29.786396 3822 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0326 23:32:29.786402 3822 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0326 23:32:29.786719 3822 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0326 23:32:29.786736 3822 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0326 23:32:29.786748 3822 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0326 23:32:29.787864 3822 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0326 23:32:29.787883 3822 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0326 23:32:29.787890 3822 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0326 23:32:29.789418 3822 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0326 23:32:29.789438 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0326 23:32:29.789449 3822 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0326 23:32:29.789983 3822 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0326 23:32:29.790000 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0326 23:32:29.790007 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0326 23:32:29.790530 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0326 23:32:29.790551 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0326 23:32:29.790559 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0326 23:32:29.791087 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:32:29.791105 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0326 23:32:29.791112 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0326 23:32:29.791417 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0326 23:32:29.791436 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0326 23:32:29.791442 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0326 23:32:29.792668 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0326 23:32:29.792687 3822 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0326 23:32:29.792703 3822 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0326 23:32:29.793212 3822 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0326 23:32:29.793233 3822 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0326 23:32:29.793244 3822 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0326 23:32:29.794555 3822 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0326 23:32:29.794572 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0326 23:32:29.794580 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0326 23:32:29.795124 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0326 23:32:29.795145 3822 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0326 23:32:29.795172 3822 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:32:29.795919 3822 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0326 23:32:29.800218 3822 egressfirewall.go:319] Starting EgressFirewall controller I0326 23:32:29.800251 3822 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0326 23:32:29.800290 3822 controller.go:133] Adding controller egress-firewall-controller event handlers I0326 23:32:29.800300 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0326 23:32:29.800312 3822 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0326 23:32:29.800319 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0326 23:32:29.800329 3822 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0326 23:32:29.800610 3822 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0326 23:32:29.800624 3822 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0326 23:32:29.800635 3822 egressqos.go:193] Setting up event handlers for EgressQoS I0326 23:32:29.800713 3822 egressfirewall.go:628] Syncing node "ip-10-0-0-231.ec2.internal" for egress firewall I0326 23:32:29.800727 3822 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:29.800778 3822 egressqos.go:245] Starting EgressQoS Controller I0326 23:32:29.800794 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0326 23:32:29.800804 3822 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0326 23:32:29.800815 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0326 23:32:29.800822 3822 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0326 23:32:29.800830 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0326 23:32:29.800838 3822 shared_informer.go:356] "Caches are synced" controller="egressqos" I0326 23:32:29.800842 3822 egressqos.go:259] Repairing EgressQoSes I0326 23:32:29.800846 3822 egressqos.go:399] Starting repairing loop for egressqos I0326 23:32:29.800923 3822 egressqos.go:401] Finished repairing loop for egressqos: 76.814µs I0326 23:32:29.800944 3822 egressservice_zone.go:132] Setting up event handlers for Egress Services I0326 23:32:29.800976 3822 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-231.ec2.internal I0326 23:32:29.801046 3822 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-231.ec2.internal : 69.286µs I0326 23:32:29.801102 3822 egressservice_zone.go:210] Starting Egress Services Controller I0326 23:32:29.801114 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:32:29.801120 3822 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:32:29.801129 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:32:29.801135 3822 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:32:29.801144 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:32:29.801150 3822 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:32:29.801155 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0326 23:32:29.801164 3822 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0326 23:32:29.801169 3822 egressservice_zone.go:228] Repairing Egress Services I0326 23:32:29.801448 3822 master_controller.go:88] Starting Admin Policy Based Route Controller I0326 23:32:29.801458 3822 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:32:29.801515 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-231.ec2.internal I0326 23:32:29.801524 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-231.ec2.internal: 11.259µs I0326 23:32:29.801532 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:29.801538 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 5.49µs I0326 23:32:29.801562 3822 default_network_controller.go:568] Completing all the Watchers took 308.186875ms I0326 23:32:29.801578 3822 default_network_controller.go:572] Starting unidling controllers I0326 23:32:29.801592 3822 unidle.go:46] Registering OVN SB ControllerEvent handler I0326 23:32:29.801602 3822 unidle.go:63] Populating Initial ContollerEvent events I0326 23:32:29.801613 3822 unidle.go:79] Setting up event handlers for services I0326 23:32:29.801661 3822 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:32:29.842316 3822 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:30.342289 3822 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:30.841729 3822 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:31.341734 3822 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:31.667645 3822 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:32:31.841892 3822 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:31.856032 3822 bridgeconfig.go:537] Gateway is ready I0326 23:32:31.856050 3822 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0326 23:32:31.865602 3822 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.231. Updating OVN encap IP. I0326 23:32:31.869775 3822 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.231 - it is already configured I0326 23:32:31.869787 3822 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.231/24:{} 172.20.0.1/32:{}]. Updating annotations. I0326 23:32:31.869981 3822 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.231/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-231.ec2.internal","mac-address":"12:f7:2e:b8:9a:8b","ip-addresses":["10.0.0.231/24"],"ip-address":"10.0.0.231/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:c3c005d2-39af-4b39-a3c0-c73a9ecd5b73 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.231/24"}] on node ip-10-0-0-231.ec2.internal I0326 23:32:31.885773 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-231.ec2.internal in network "default" I0326 23:32:31.885814 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-231.ec2.internal I0326 23:32:31.885823 3822 master.go:532] Adding or Updating local node "ip-10-0-0-231.ec2.internal" for network "default" I0326 23:32:31.885855 3822 egressfirewall.go:628] Syncing node "ip-10-0-0-231.ec2.internal" for egress firewall I0326 23:32:31.885863 3822 node_tracker.go:169] Node ip-10-0-0-231.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:31.885894 3822 services_controller.go:574] Full service sync requested for network=default I0326 23:32:31.885826 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-231.ec2.internal: 20.635µs I0326 23:32:31.886091 3822 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.231 I0326 23:32:31.886180 3822 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-231.ec2.internal I0326 23:32:31.890145 3822 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-231.ec2.internal. Updating no re-route policies I0326 23:32:31.890234 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-x5dth I0326 23:32:31.890242 3822 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-lhmzk I0326 23:32:31.890245 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-pc976 I0326 23:32:31.890253 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-x5dth I0326 23:32:31.890272 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-pc976 I0326 23:32:31.890275 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-vshpw I0326 23:32:31.890289 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-vshpw I0326 23:32:31.890287 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-99h22 I0326 23:32:31.890287 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-231.ec2.internal I0326 23:32:31.890304 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-99h22 I0326 23:32:31.890306 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-231.ec2.internal I0326 23:32:31.890302 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nz7mp I0326 23:32:31.890233 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-4wfbf I0326 23:32:31.890315 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-x5dth after 0 failed attempt(s) I0326 23:32:31.890313 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nfmjv I0326 23:32:31.890322 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-231.ec2.internal after 0 failed attempt(s) I0326 23:32:31.890325 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-pc976 after 0 failed attempt(s) I0326 23:32:31.890253 3822 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-lhmzk I0326 23:32:31.890330 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nfmjv I0326 23:32:31.890330 3822 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-vshpw, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-vshpw: suppressed error logged: pod openshift-network-diagnostics/network-check-target-vshpw: no pod IPs found I0326 23:32:31.890333 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-qp5hd I0326 23:32:31.890331 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-hdrb2 I0326 23:32:31.890337 3822 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-231.ec2.internal I0326 23:32:31.890340 3822 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-lhmzk after 0 failed attempt(s) I0326 23:32:31.890340 3822 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-z9lfw I0326 23:32:31.890342 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-qp5hd I0326 23:32:31.890345 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-hdrb2 I0326 23:32:31.890343 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-9b2h5 I0326 23:32:31.890350 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-zshc2 I0326 23:32:31.890360 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-qp5hd after 0 failed attempt(s) I0326 23:32:31.890360 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-zshc2 I0326 23:32:31.890365 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-hdrb2 after 0 failed attempt(s) I0326 23:32:31.890367 3822 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-nfmjv] creating logical port openshift-multus_network-metrics-daemon-nfmjv for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:31.890380 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-zshc2 after 0 failed attempt(s) I0326 23:32:31.890337 3822 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-99h22] creating logical port openshift-network-diagnostics_network-check-target-99h22 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:31.890319 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nz7mp I0326 23:32:31.890458 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nz7mp after 0 failed attempt(s) I0326 23:32:31.890330 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-4wfbf I0326 23:32:31.890360 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-9b2h5 I0326 23:32:31.890502 3822 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-4wfbf, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-4wfbf: suppressed error logged: pod openshift-multus/network-metrics-daemon-4wfbf: no pod IPs found I0326 23:32:31.890504 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-9b2h5 after 0 failed attempt(s) I0326 23:32:31.890344 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-z98jg I0326 23:32:31.890527 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-z98jg I0326 23:32:31.890545 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-z98jg after 0 failed attempt(s) I0326 23:32:31.890346 3822 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-231.ec2.internal I0326 23:32:31.890563 3822 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-231.ec2.internal after 0 failed attempt(s) I0326 23:32:31.890349 3822 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-z9lfw I0326 23:32:31.890587 3822 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-z9lfw] creating logical port kube-system_global-pull-secret-syncer-z9lfw for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:31.891217 3822 pods.go:271] [openshift-network-diagnostics/network-check-target-99h22] addLogicalPort took 887.515µs, libovsdb time 431.504µs I0326 23:32:31.891230 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-99h22 after 0 failed attempt(s) I0326 23:32:31.891332 3822 pods.go:271] [openshift-multus/network-metrics-daemon-nfmjv] addLogicalPort took 977.595µs, libovsdb time 504.718µs I0326 23:32:31.891340 3822 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-nfmjv after 0 failed attempt(s) I0326 23:32:31.891425 3822 pods.go:271] [kube-system/global-pull-secret-syncer-z9lfw] addLogicalPort took 845.155µs, libovsdb time 481.84µs I0326 23:32:31.891433 3822 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-z9lfw after 0 failed attempt(s) I0326 23:32:31.906707 3822 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0326 23:32:31.916961 3822 iptables.go:108] Creating table: mangle chain: OUTPUT I0326 23:32:31.919971 3822 iptables.go:108] Creating table: nat chain: OUTPUT I0326 23:32:31.950073 3822 iptables.go:108] Creating table: nat chain: PREROUTING I0326 23:32:32.006361 3822 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:32:32.022107 3822 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0326 23:32:32.022230 3822 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0326 23:32:32.022293 3822 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0326 23:32:32.022373 3822 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0326 23:32:32.022424 3822 gateway_init.go:325] Gateway and management port readiness took 2.684123544s I0326 23:32:32.022450 3822 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0326 23:32:32.022482 3822 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0326 23:32:32.022493 3822 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0326 23:32:32.022500 3822 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:32:32.022552 3822 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0326 23:32:32.022655 3822 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 57.284µs I0326 23:32:32.022865 3822 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0326 23:32:32.022890 3822 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 16.166µs I0326 23:32:32.022898 3822 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.084µs I0326 23:32:32.022904 3822 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0326 23:32:32.022912 3822 nad_controller.go:173] [node-nad-controller NAD controller]: started I0326 23:32:32.022916 3822 default_node_network_controller.go:898] Starting the default node network controller I0326 23:32:32.022921 3822 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0326 23:32:32.023006 3822 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 11.521µs I0326 23:32:32.023090 3822 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0326 23:32:32.023100 3822 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0326 23:32:32.023126 3822 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a1f310 0xc0002c7a40 0xc0002c7ab0 0xc0002c7b90 0xc0002c7c70 0xc0002c7ce0 0xc0002c7d50 0xc000a72f50 0xc0002c7dc0 0xc0002c7e30 0xc000644410 0xc0061d1180 0xc0002c7ea0 map[0x2f67040:0xc0047fe300 0x2f67580:0xc0047fe480 0x2f68000:0xc002a91e00 0x2f682a0:0xc0047fe2a0 0x2f687e0:0xc0047fe360 0x2f68a80:0xc0047fe3c0 0x2f68d20:0xc0047fe420 0x2f69500:0xc0047fe4e0 0x2f697a0:0xc0047fe540 0x2f93d40:0xc000ab5200 0x2f940e0:0xc000ab5260 0x2f94bc0:0xc000c20f00 0x2f96520:0xc001c7e000 0x2f97000:0xc00260e600 0x2f97e80:0xc001c7e060] 0xc0001753b0 0} I0326 23:32:32.023179 3822 gateway.go:151] Starting gateway service sync I0326 23:32:32.023615 3822 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0326 23:32:32.025604 3822 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0326 23:32:32.060645 3822 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0326 23:32:32.061761 3822 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0326 23:32:32.062765 3822 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0326 23:32:32.094723 3822 gateway.go:168] Gateway service sync done. Time taken: 71.530614ms I0326 23:32:32.095281 3822 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32538 I0326 23:32:32.095556 3822 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a1f310 0xc0002c7a40 0xc0002c7ab0 0xc0002c7b90 0xc0002c7c70 0xc0002c7ce0 0xc0002c7d50 0xc000a72f50 0xc0002c7dc0 0xc0002c7e30 0xc000644410 0xc0061d1180 0xc0002c7ea0 map[0x2f67040:0xc0047fe300 0x2f67580:0xc0047fe480 0x2f68000:0xc002a91e00 0x2f682a0:0xc0047fe2a0 0x2f687e0:0xc0047fe360 0x2f68a80:0xc0047fe3c0 0x2f68d20:0xc0047fe420 0x2f69500:0xc0047fe4e0 0x2f697a0:0xc0047fe540 0x2f93d40:0xc000ab5200 0x2f940e0:0xc000ab5260 0x2f94bc0:0xc000c20f00 0x2f96520:0xc001c7e000 0x2f97000:0xc00260e600 0x2f97e80:0xc001c7e060] 0xc0001753b0 0} I0326 23:32:32.095994 3822 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a1f310 0xc0002c7a40 0xc0002c7ab0 0xc0002c7b90 0xc0002c7c70 0xc0002c7ce0 0xc0002c7d50 0xc000a72f50 0xc0002c7dc0 0xc0002c7e30 0xc000644410 0xc0061d1180 0xc0002c7ea0 map[0x2f67040:0xc0047fe300 0x2f67580:0xc0047fe480 0x2f68000:0xc002a91e00 0x2f682a0:0xc0047fe2a0 0x2f687e0:0xc0047fe360 0x2f68a80:0xc0047fe3c0 0x2f68d20:0xc0047fe420 0x2f69500:0xc0047fe4e0 0x2f697a0:0xc0047fe540 0x2f93d40:0xc000ab5200 0x2f940e0:0xc000ab5260 0x2f94bc0:0xc000c20f00 0x2f96520:0xc001c7e000 0x2f97000:0xc00260e600 0x2f97e80:0xc001c7e060] 0xc0001753b0 0} I0326 23:32:32.096025 3822 gateway.go:519] Reconciling gateway with updates I0326 23:32:32.101094 3822 gateway.go:326] Spawning Conntrack Rule Check Thread I0326 23:32:32.101123 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:32.101128 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:32.101140 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:32.101139 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:32.101145 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:32.101154 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:32.101155 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:32:32.101159 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:32.101171 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:32.101176 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:32.101176 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:32.101186 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:32:32.101179 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:32.101193 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:32.101202 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:32.101205 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:32.101211 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:32.101212 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:32:32.101219 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:32:32.101220 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:32.101226 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:32:32.101232 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:32:32.101233 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:32.101239 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:32:32.101244 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:32.101247 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:32:32.101244 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:32.101243 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:32.101254 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:32.101254 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:32:32.101277 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:32.101278 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:32.101276 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:32.101285 3822 default_node_network_controller.go:943] Gateway and management port readiness took 78.177364ms I0326 23:32:32.101291 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:32.101296 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:32.101297 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:32.101300 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:32:32.101301 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:32:32.101307 3822 port_claim.go:71] Svc openshift-ingress/router-default: port 32508 is already open, no action needed I0326 23:32:32.101307 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:32.101191 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:32.101317 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:32.101232 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:32.101323 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:32.101299 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:32:32.101330 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:32.101332 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:32.101338 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:32.101339 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:32.101344 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:32.101343 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:32.101193 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:32:32.101354 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:32.101355 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:32.101342 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:32.101366 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:32:32.101316 3822 port_claim.go:71] Svc openshift-ingress/router-default: port 30435 is already open, no action needed I0326 23:32:32.101376 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:32.101324 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:32.101389 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:32.101400 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:32:32.101388 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:32.101418 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:32.101319 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:32.101426 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:32.101426 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:32.101431 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:32.101433 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:32.101435 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:32.101438 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:32.101326 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:32.101450 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:32.101461 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:32:32.101429 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:32.101233 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:32:32.101333 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:32.101453 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:32.101437 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:32.101425 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:32.101302 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:32.101545 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:32.101551 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:32.101336 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:32.101570 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:32.101341 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:32.101552 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:32.101585 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:32.101589 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:32:32.101244 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:32.101602 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:32.101316 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:32.101635 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:32.101415 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:32:32.101649 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:32:32.101321 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:32.101667 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:32.101437 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:32.101710 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:32.101440 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:32:32.101440 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:32.101330 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:32.101343 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:32.101800 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:32.101348 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:32.101839 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:32:32.101355 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:32.102201 3822 node_ip_handler_linux.go:160] Node IP manager is running W0326 23:32:32.135636 3822 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:32:32.135809 3822 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:32:32.135931 3822 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0326 23:32:32.144296 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:32:32.151636 3822 default_network_controller.go:1008] Node "ip-10-0-2-151.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:32.162847 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:32:32.162856 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:32:32.162871 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:32:32.162881 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:32:32.162891 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:32:32.162896 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:32:32.162904 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:32:32.162913 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:32:32.198859 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:32:32.198868 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:32:32.198883 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:32:32.198891 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:32:32.198899 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:32:32.198906 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:32:32.198912 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:32:32.198918 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:32:32.198927 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:32:32.198933 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:32:32.198939 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:32:32.198946 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:32:32.198951 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:32:32.198956 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:32:32.198980 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:32:32.198988 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:32:32.198993 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:32:32.198998 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:32:32.199003 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:32:32.351513 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:32.351544 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 32.346µs I0326 23:32:32.351549 3822 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:32.351594 3822 default_network_controller.go:1008] Node "ip-10-0-2-151.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:33.469837 3822 default_node_network_controller.go:1364] Starting node controller node sync W0326 23:32:33.469887 3822 egressip_healthcheck.go:76] Health checking using insecure connection I0326 23:32:33.469938 3822 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0326 23:32:33.481232 3822 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.37291ms I0326 23:32:33.481451 3822 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0326 23:32:33.481725 3822 egressservice_node.go:93] Setting up event handlers for Egress Services I0326 23:32:33.481931 3822 egressservice_node.go:183] Starting Egress Services Controller I0326 23:32:33.481946 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:32:33.481954 3822 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:32:33.481959 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:32:33.481963 3822 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:32:33.481969 3822 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:32:33.481973 3822 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:32:33.481996 3822 egressservice_node.go:197] Repairing Egress Services I0326 23:32:33.491780 3822 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0326 23:32:33.491792 3822 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:32:33.492429 3822 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0326 23:32:33.492445 3822 link_network_manager.go:117] Link manager is running I0326 23:32:33.492454 3822 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0326 23:32:33.492491 3822 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0326 23:32:33.492743 3822 vrf_manager.go:128] VRF manager is running I0326 23:32:33.492833 3822 node_controller_manager.go:437] Removing flows to drop GARP I0326 23:32:33.492842 3822 gateway.go:519] Reconciling gateway with updates I0326 23:32:33.516185 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:33.516195 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:33.516203 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:33.516183 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:33.516214 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:33.516214 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:33.516226 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:33.516234 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:32:33.516223 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:33.516235 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:33.516245 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:32:33.516246 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:32:33.516256 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:33.516272 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:33.516278 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:32:33.516278 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:33.516286 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:32:33.516291 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:33.516293 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:33.516302 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:33.516299 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:33.516310 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:33.516314 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:32:33.516204 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:33.516318 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:32:33.516320 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:33.516333 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:33.516336 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:33.516340 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:33.516346 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:32:33.516216 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:33.516372 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:33.516387 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:32:33.516394 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:33.516399 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:32:33.516399 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:33.516401 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:32:33.516409 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:32:33.516408 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:33.516417 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:32:33.516420 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:33.516425 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:33.516424 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:33.516429 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:33.516434 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:33.516438 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:33.516444 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:33.516447 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:33.516455 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:32:33.516454 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:33.516462 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:33.516468 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:32:33.516475 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:33.516467 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:33.516487 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:32:33.516443 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:33.516492 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:32:33.516499 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:32:33.516278 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:33.516509 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:32:33.516515 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:33.516521 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:32:33.516520 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:32:33.516226 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:33.516552 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:33.516302 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:33.516572 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:32:33.516374 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:33.516600 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:33.516601 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:33.516612 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:33.516618 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:33.516618 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:33.516622 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:33.516625 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:32:33.516630 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:33.516247 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:33.516640 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:33.516234 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:33.516406 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:33.516389 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:33.516319 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:33.516243 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:33.516209 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:33.516216 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:33.516216 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:33.516815 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:33.516207 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:33.516235 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:33.516871 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:33.516281 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:33.516302 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:33.516943 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:33.516324 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:33.516387 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:33.517014 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:33.516389 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:33.517050 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:33.516402 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:32:33.516408 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:33.517091 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:33.517107 3822 port_claim.go:71] Svc openshift-ingress/router-default: port 32508 is already open, no action needed I0326 23:32:33.517115 3822 port_claim.go:71] Svc openshift-ingress/router-default: port 30435 is already open, no action needed I0326 23:32:33.516416 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:33.517160 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:32:33.516425 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:33.517173 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:33.516429 3822 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:33.517211 3822 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:33.516631 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:32:33.559085 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:32:33.559110 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:32:33.559117 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:32:33.599624 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:32:33.599651 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:32:33.599658 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:32:33.599665 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:32:33.599672 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:32:33.599678 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:32:33.599684 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:32:33.599689 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:32:33.599696 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:32:33.599701 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:32:33.599708 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:32:33.599716 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:32:33.599721 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:32:33.599727 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:32:33.599734 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:32:33.599745 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:32:33.599752 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:32:33.599759 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:32:33.642692 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:32:33.642719 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:32:33.642737 3822 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:32:35.725616 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-231.ec2.internal I0326 23:32:35.725645 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-231.ec2.internal: 36.091µs I0326 23:32:35.752904 3822 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5fd5865cf4-s6vd6] creating logical port openshift-image-registry_image-registry-5fd5865cf4-s6vd6 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:35.753052 3822 kube.go:256] Updating pod openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:32:35.775512 3822 pod.go:62] [openshift-image-registry/image-registry-5fd5865cf4-s6vd6] pod update took 22.48929ms I0326 23:32:35.775541 3822 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-5fd5865cf4-s6vd6] addLogicalPort annotation time took 22.529832ms I0326 23:32:35.777068 3822 pods.go:271] [openshift-image-registry/image-registry-5fd5865cf4-s6vd6] addLogicalPort took 24.182925ms, libovsdb time 1.179928ms I0326 23:32:35.791636 3822 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-zp846] creating logical port openshift-dns_dns-default-zp846 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:35.791747 3822 kube.go:256] Updating pod openshift-dns/dns-default-zp846 I0326 23:32:35.793022 3822 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-566676bfc5-57vrn] creating logical port openshift-ingress_router-default-566676bfc5-57vrn for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:35.793152 3822 kube.go:256] Updating pod openshift-ingress/router-default-566676bfc5-57vrn I0326 23:32:35.809140 3822 pod.go:62] [openshift-dns/dns-default-zp846] pod update took 17.416421ms I0326 23:32:35.809418 3822 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-zp846] addLogicalPort annotation time took 17.699009ms I0326 23:32:35.810947 3822 pods.go:271] [openshift-dns/dns-default-zp846] addLogicalPort took 19.324521ms, libovsdb time 1.278786ms I0326 23:32:35.812650 3822 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576130-zs7r9 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:35.812774 3822 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 I0326 23:32:35.816486 3822 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-s6gvl for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:35.816623 3822 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl I0326 23:32:35.816694 3822 pod.go:62] [openshift-ingress/router-default-566676bfc5-57vrn] pod update took 23.582056ms I0326 23:32:35.816719 3822 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-566676bfc5-57vrn] addLogicalPort annotation time took 23.606331ms I0326 23:32:35.817969 3822 pods.go:271] [openshift-ingress/router-default-566676bfc5-57vrn] addLogicalPort took 24.958463ms, libovsdb time 921.385µs I0326 23:32:35.818017 3822 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-flpsb] creating logical port openshift-ingress-canary_ingress-canary-flpsb for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:35.818070 3822 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-flpsb I0326 23:32:35.825976 3822 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9] pod update took 13.223441ms I0326 23:32:35.825991 3822 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9] addLogicalPort annotation time took 13.242641ms I0326 23:32:35.826906 3822 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9] addLogicalPort took 14.266239ms, libovsdb time 695.327µs I0326 23:32:35.840414 3822 pod.go:62] [openshift-ingress-canary/ingress-canary-flpsb] pod update took 22.356469ms I0326 23:32:35.840433 3822 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-flpsb] addLogicalPort annotation time took 22.37892ms I0326 23:32:35.840531 3822 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl] pod update took 23.943572ms I0326 23:32:35.840594 3822 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl] addLogicalPort annotation time took 24.007467ms I0326 23:32:35.841516 3822 pods.go:271] [openshift-ingress-canary/ingress-canary-flpsb] addLogicalPort took 23.510152ms, libovsdb time 815.418µs I0326 23:32:35.841993 3822 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl] addLogicalPort took 25.51905ms, libovsdb time 1.104837ms I0326 23:32:36.525705 3822 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default] I0326 23:32:36.525841 3822 cni.go:403] [openshift-ingress-canary/ingress-canary-flpsb 9b34535ba20d371fb09a05110d8cbd8870751f770ea897a38ef72c4503c52131 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-flpsb 9b34535ba20d371fb09a05110d8cbd8870751f770ea897a38ef72c4503c52131 network default NAD default] I0326 23:32:36.526493 3822 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl 98d5c8e8d723ae25b52f66596fcb92d7664bc5d63c8ffde3af487ed9a5c932d7 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl 98d5c8e8d723ae25b52f66596fcb92d7664bc5d63c8ffde3af487ed9a5c932d7 network default NAD default] I0326 23:32:36.584519 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-flpsb, hostIfaceName: 9b34535ba20d371, network: default, NAD default, SandboxID: "9b34535ba20d371fb09a05110d8cbd8870751f770ea897a38ef72c4503c52131", PCI device ID: , UID: "c35c9191-0352-4a1d-ab92-bbd3ae40a803", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0326 23:32:36.589535 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576130-zs7r9, hostIfaceName: 3d99a71112b59bd, network: default, NAD default, SandboxID: "3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95", PCI device ID: , UID: "f83ab361-40ed-4574-bbb4-028bce25b6c0", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0326 23:32:36.590937 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-s6gvl, hostIfaceName: 98d5c8e8d723ae2, network: default, NAD default, SandboxID: "98d5c8e8d723ae25b52f66596fcb92d7664bc5d63c8ffde3af487ed9a5c932d7", PCI device ID: , UID: "f2f70af2-ff36-4583-9a39-bd0958c53908", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0326 23:32:36.641120 3822 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:36:fc:60:55:10\",\"name\":\"3d99a71112b59bd\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29bb5957-0131-4673-8301-257b0bf93b37\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:36.641182 3822 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl 98d5c8e8d723ae25b52f66596fcb92d7664bc5d63c8ffde3af487ed9a5c932d7 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s6gvl 98d5c8e8d723ae25b52f66596fcb92d7664bc5d63c8ffde3af487ed9a5c932d7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:7d:52:d8:0d:7f\",\"name\":\"98d5c8e8d723ae2\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c56a498-25ab-4927-9f1a-1280fbda5c73\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:36.641194 3822 cni.go:424] [openshift-ingress-canary/ingress-canary-flpsb 9b34535ba20d371fb09a05110d8cbd8870751f770ea897a38ef72c4503c52131 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-flpsb 9b34535ba20d371fb09a05110d8cbd8870751f770ea897a38ef72c4503c52131 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:d2:b6:d2:b0:53\",\"name\":\"9b34535ba20d371\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8bf2d24-7582-4cd3-823c-729741241580\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:36.871123 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-151.ec2.internal in network "default" I0326 23:32:36.871127 3822 default_network_controller.go:1008] Node "ip-10-0-2-151.ec2.internal" in remote zone "ip-10-0-2-151.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:36.871172 3822 node_tracker.go:235] Node ip-10-0-2-151.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-151.ec2.internal" I0326 23:32:36.871189 3822 node_tracker.go:169] Node ip-10-0-2-151.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:36.871206 3822 services_controller.go:574] Full service sync requested for network=default I0326 23:32:36.961015 3822 default_network_controller.go:1008] Node "ip-10-0-2-151.ec2.internal" in remote zone "ip-10-0-2-151.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:36.992155 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-151.ec2.internal in network "default" I0326 23:32:36.992229 3822 node_tracker.go:169] Node ip-10-0-2-151.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:36.992246 3822 services_controller.go:574] Full service sync requested for network=default I0326 23:32:36.992251 3822 default_network_controller.go:1008] Node "ip-10-0-2-151.ec2.internal" in remote zone "ip-10-0-2-151.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:36.993047 3822 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-151.ec2.internal for the network default I0326 23:32:36.994214 3822 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-151.ec2.internal" on network "default" took: 1.180514ms I0326 23:32:38.746098 3822 cni.go:403] [kube-system/global-pull-secret-syncer-z9lfw c18abb88da49f96c58cc761edd2f582af980062276d86fce553692f0583041c5 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-z9lfw c18abb88da49f96c58cc761edd2f582af980062276d86fce553692f0583041c5 network default NAD default] I0326 23:32:38.754841 3822 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-z9lfw, hostIfaceName: c18abb88da49f96, network: default, NAD default, SandboxID: "c18abb88da49f96c58cc761edd2f582af980062276d86fce553692f0583041c5", PCI device ID: , UID: "bcac3e3c-0370-460f-8263-4347dbac5bf3", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0326 23:32:38.793318 3822 cni.go:424] [kube-system/global-pull-secret-syncer-z9lfw c18abb88da49f96c58cc761edd2f582af980062276d86fce553692f0583041c5 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-z9lfw c18abb88da49f96c58cc761edd2f582af980062276d86fce553692f0583041c5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:3e:66:ea:f0:e0\",\"name\":\"c18abb88da49f96\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c6b7d0e-e349-476b-aede-9400dc1ee3ba\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:38.959049 3822 cni.go:403] [openshift-network-diagnostics/network-check-target-99h22 b0a992fe928ea4156799b0a2ba16a7600ba5b88fe8efa4535f979cd395753a15 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-99h22 b0a992fe928ea4156799b0a2ba16a7600ba5b88fe8efa4535f979cd395753a15 network default NAD default] I0326 23:32:38.967913 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-99h22, hostIfaceName: b0a992fe928ea41, network: default, NAD default, SandboxID: "b0a992fe928ea4156799b0a2ba16a7600ba5b88fe8efa4535f979cd395753a15", PCI device ID: , UID: "f141a5b4-2dfd-44cd-882e-91b56dbb4bd5", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0326 23:32:39.006922 3822 cni.go:424] [openshift-network-diagnostics/network-check-target-99h22 b0a992fe928ea4156799b0a2ba16a7600ba5b88fe8efa4535f979cd395753a15 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-99h22 b0a992fe928ea4156799b0a2ba16a7600ba5b88fe8efa4535f979cd395753a15 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:ea:12:55:74:7e\",\"name\":\"b0a992fe928ea41\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cb260fc-d7de-4dce-b1b2-3cf70434f6d6\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:39.544960 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-151.ec2.internal in network "default" I0326 23:32:39.544991 3822 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:39.545023 3822 node_tracker.go:169] Node ip-10-0-2-151.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:39.545036 3822 services_controller.go:574] Full service sync requested for network=default I0326 23:32:39.544961 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:39.545072 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 113.382µs I0326 23:32:39.544967 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-151.ec2.internal", IP: 10.0.2.151 to PMTUD blocking rules I0326 23:32:39.545089 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-151.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:32:39.551052 3822 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-151.ec2.internal. Updating no re-route policies I0326 23:32:39.563054 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-151.ec2.internal", IP: 10.0.2.151 to PMTUD blocking rules I0326 23:32:39.563069 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-151.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:32:42.480781 3822 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:42.481242 3822 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:43.132502 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:43.132530 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 33.234µs I0326 23:32:43.539056 3822 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:43.539080 3822 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:32:47.079127 3822 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:32:47.126631 3822 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0326 23:32:49.500240 3822 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:32:49.853572 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:32:49.853596 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 31.9µs E0326 23:32:49.853627 3822 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0326 23:32:49.853638 3822 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:32:49.853653 3822 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-147.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-147.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-147.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-147.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-147.ec2.internal, error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] I0326 23:32:49.853631 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:32:49.853724 3822 node_tracker.go:220] Node ip-10-0-1-147.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0326 23:32:49.867949 3822 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input I0326 23:32:49.867966 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0326 23:32:49.867995 3822 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0326 23:32:49.879799 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:49.879811 3822 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:49.879819 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 22.241µs I0326 23:32:49.882290 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:49.882301 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:32:49.882313 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 15.217µs E0326 23:32:49.882327 3822 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0326 23:32:49.882297 3822 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall E0326 23:32:49.882297 3822 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input E0326 23:32:49.960428 3822 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input E0326 23:32:49.960454 3822 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input I0326 23:32:49.960475 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0326 23:32:49.960526 3822 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-147.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-147.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-147.ec2.internal" I0326 23:32:49.971435 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0326 23:32:50.055255 3822 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:32:50.055273 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:32:50.055300 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 45.208µs I0326 23:32:50.055273 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:50.067693 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:50.317662 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:52.155879 3822 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-tsgmk for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:52.155968 3822 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk I0326 23:32:52.168653 3822 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk] pod update took 12.707786ms I0326 23:32:52.168674 3822 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk] addLogicalPort annotation time took 12.734331ms I0326 23:32:52.169838 3822 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk] addLogicalPort took 13.989819ms, libovsdb time 894.941µs I0326 23:32:52.547424 3822 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk 85756945965b3e5d19c109180baf3c72973ff30530eb7fadc16b0e36069b1180 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk 85756945965b3e5d19c109180baf3c72973ff30530eb7fadc16b0e36069b1180 network default NAD default] I0326 23:32:52.558103 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-tsgmk, hostIfaceName: 85756945965b3e5, network: default, NAD default, SandboxID: "85756945965b3e5d19c109180baf3c72973ff30530eb7fadc16b0e36069b1180", PCI device ID: , UID: "49e41ec3-dc78-4758-84f7-f39e2c616fe3", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0326 23:32:52.563587 3822 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-49ptw for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:52.563739 3822 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw I0326 23:32:52.566419 3822 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-j59gl for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:52.566521 3822 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl I0326 23:32:52.583516 3822 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl] pod update took 17.024231ms I0326 23:32:52.583535 3822 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl] addLogicalPort annotation time took 17.047817ms I0326 23:32:52.584707 3822 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl] addLogicalPort took 18.30561ms, libovsdb time 921.287µs I0326 23:32:52.587535 3822 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw] pod update took 23.825663ms I0326 23:32:52.587552 3822 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw] addLogicalPort annotation time took 23.85318ms I0326 23:32:52.588840 3822 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw] addLogicalPort took 25.271972ms, libovsdb time 1.019011ms I0326 23:32:52.599667 3822 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk 85756945965b3e5d19c109180baf3c72973ff30530eb7fadc16b0e36069b1180 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-tsgmk 85756945965b3e5d19c109180baf3c72973ff30530eb7fadc16b0e36069b1180 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:43:39:37:58:a8\",\"name\":\"85756945965b3e5\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2767a254-f08e-43fa-a116-ddb3401c68bc\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:52.948875 3822 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw 38cf896919c4f724a28e27fd6bb24419a96a70473da3226b45d321d9e24b0ac3 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw 38cf896919c4f724a28e27fd6bb24419a96a70473da3226b45d321d9e24b0ac3 network default NAD default] I0326 23:32:52.958773 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-49ptw, hostIfaceName: 38cf896919c4f72, network: default, NAD default, SandboxID: "38cf896919c4f724a28e27fd6bb24419a96a70473da3226b45d321d9e24b0ac3", PCI device ID: , UID: "6e7ae030-d69b-4c4e-b92b-7420df5dcc47", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0326 23:32:52.970579 3822 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl be22d701dce6b5637e5e9ff06286a4f535269bedab3e79430c159d6c2405a08b network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl be22d701dce6b5637e5e9ff06286a4f535269bedab3e79430c159d6c2405a08b network default NAD default] I0326 23:32:52.982220 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-j59gl, hostIfaceName: be22d701dce6b56, network: default, NAD default, SandboxID: "be22d701dce6b5637e5e9ff06286a4f535269bedab3e79430c159d6c2405a08b", PCI device ID: , UID: "80ac9cd4-7a09-4a1c-9849-793e08805be0", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0326 23:32:53.002437 3822 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw 38cf896919c4f724a28e27fd6bb24419a96a70473da3226b45d321d9e24b0ac3 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-49ptw 38cf896919c4f724a28e27fd6bb24419a96a70473da3226b45d321d9e24b0ac3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:5e:ed:ca:1b:8e\",\"name\":\"38cf896919c4f72\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3f7a3ba0-2cfe-473a-9e38-362ade7ae32f\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:53.020558 3822 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl be22d701dce6b5637e5e9ff06286a4f535269bedab3e79430c159d6c2405a08b network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-j59gl be22d701dce6b5637e5e9ff06286a4f535269bedab3e79430c159d6c2405a08b network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:d2:66:07:5d:1c\",\"name\":\"be22d701dce6b56\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/deaca271-d91f-4417-8d0b-1b615987c166\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:54.227082 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:32:57.128988 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:57.186951 3822 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-k62rj] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-k62rj for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:57.187086 3822 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-k62rj I0326 23:32:57.204230 3822 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-k62rj] pod update took 17.172606ms I0326 23:32:57.204271 3822 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-k62rj] addLogicalPort annotation time took 17.202651ms I0326 23:32:57.205415 3822 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-k62rj] addLogicalPort took 18.482404ms, libovsdb time 800.725µs I0326 23:32:58.000302 3822 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0326 23:32:58.001178 3822 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 853.404µs I0326 23:32:58.049805 3822 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:32:58.122934 3822 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-qm22g for pod on switch ip-10-0-0-231.ec2.internal I0326 23:32:58.123054 3822 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g I0326 23:32:58.140092 3822 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g] pod update took 17.081644ms I0326 23:32:58.140113 3822 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g] addLogicalPort annotation time took 17.109704ms I0326 23:32:58.141346 3822 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g] addLogicalPort took 18.425333ms, libovsdb time 886.421µs I0326 23:32:58.458230 3822 namespace.go:98] [openshift-service-ca] adding namespace I0326 23:32:58.460039 3822 namespace.go:102] [openshift-service-ca] adding namespace took 1.786346ms I0326 23:32:58.480152 3822 namespace.go:142] [openshift-service-ca] updating namespace I0326 23:32:58.502057 3822 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g 1320cb760b55d0822fe33406392d50cb318bafaf12a1642b37d4d969cb191cca network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g 1320cb760b55d0822fe33406392d50cb318bafaf12a1642b37d4d969cb191cca network default NAD default] I0326 23:32:58.511914 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-qm22g, hostIfaceName: 1320cb760b55d08, network: default, NAD default, SandboxID: "1320cb760b55d0822fe33406392d50cb318bafaf12a1642b37d4d969cb191cca", PCI device ID: , UID: "4994c6a5-371a-4bfb-af62-8704fa4bc913", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] W0326 23:32:58.538842 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:32:58.553486 3822 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g 1320cb760b55d0822fe33406392d50cb318bafaf12a1642b37d4d969cb191cca network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-qm22g 1320cb760b55d0822fe33406392d50cb318bafaf12a1642b37d4d969cb191cca network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:5e:db:6d:f3:57\",\"name\":\"1320cb760b55d08\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0eff7bd9-5ccd-49ea-92a6-4b995630680f\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:32:59.537937 3822 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-1-147.ec2.internal I0326 23:32:59.537961 3822 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-1-147.ec2.internal I0326 23:32:59.538047 3822 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-1-147.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-147.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-147.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-147.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-147.ec2.internal I0326 23:32:59.919116 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:01.891951 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-pwvv2 I0326 23:33:01.892005 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-pwvv2 I0326 23:33:01.892050 3822 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-pwvv2, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-pwvv2: suppressed error logged: pod openshift-network-diagnostics/network-check-target-pwvv2: no pod IPs found I0326 23:33:01.891962 3822 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-r7bdt I0326 23:33:01.892062 3822 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-r7bdt I0326 23:33:01.892078 3822 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-r7bdt, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-r7bdt: suppressed error logged: pod openshift-multus/network-metrics-daemon-r7bdt: no pod IPs found W0326 23:33:07.873507 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:33:08.010823 3822 cni.go:403] [openshift-dns/dns-default-zp846 203cee20e75d35d851f250022a5cdc49e91338c5b3530a26d48e47d70a61e63c network default NAD default] ADD starting CNI request [openshift-dns/dns-default-zp846 203cee20e75d35d851f250022a5cdc49e91338c5b3530a26d48e47d70a61e63c network default NAD default] I0326 23:33:08.018867 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-zp846, hostIfaceName: 203cee20e75d35d, network: default, NAD default, SandboxID: "203cee20e75d35d851f250022a5cdc49e91338c5b3530a26d48e47d70a61e63c", PCI device ID: , UID: "a9563b47-35f4-4962-928b-8fe5fa7d23cd", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0326 23:33:08.060619 3822 cni.go:424] [openshift-dns/dns-default-zp846 203cee20e75d35d851f250022a5cdc49e91338c5b3530a26d48e47d70a61e63c network default NAD default] ADD finished CNI request [openshift-dns/dns-default-zp846 203cee20e75d35d851f250022a5cdc49e91338c5b3530a26d48e47d70a61e63c network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:a8:db:74:00:61\",\"name\":\"203cee20e75d35d\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/56452d30-6fe7-43f4-96be-1a3394cd41e5\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:11.777562 3822 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:33:11.777579 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:33:11.777600 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 27.974µs I0326 23:33:11.777566 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:13.194846 3822 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-k62rj 4410b6f98ad1289f88f2451d984147749aaff851328f16672114b71848bb319f network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-k62rj 4410b6f98ad1289f88f2451d984147749aaff851328f16672114b71848bb319f network default NAD default] I0326 23:33:13.202947 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-k62rj, hostIfaceName: 4410b6f98ad1289, network: default, NAD default, SandboxID: "4410b6f98ad1289f88f2451d984147749aaff851328f16672114b71848bb319f", PCI device ID: , UID: "63258e46-c380-472b-b187-952ee81dd925", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0326 23:33:13.242173 3822 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-k62rj 4410b6f98ad1289f88f2451d984147749aaff851328f16672114b71848bb319f network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-k62rj 4410b6f98ad1289f88f2451d984147749aaff851328f16672114b71848bb319f network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:cc:d4:e5:eb:ea\",\"name\":\"4410b6f98ad1289\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8bb8980c-cb70-4d73-8078-6e4c3a448f9c\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:20.394626 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:24.344476 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:33:24.348495 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:33:24.361514 3822 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:24.399133 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:33:24.453618 3822 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-59bb6d878f-phssf IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-59bb6d878f-phssf: no pod IPs found I0326 23:33:24.467252 3822 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-669c9bbc85-6sbq9] creating logical port openshift-ingress_router-default-669c9bbc85-6sbq9 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:33:24.467282 3822 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-59svj] creating logical port openshift-insights_insights-runtime-extractor-59svj for pod on switch ip-10-0-0-231.ec2.internal I0326 23:33:24.467397 3822 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-59svj I0326 23:33:24.467397 3822 kube.go:256] Updating pod openshift-ingress/router-default-669c9bbc85-6sbq9 I0326 23:33:24.500241 3822 pod.go:62] [openshift-ingress/router-default-669c9bbc85-6sbq9] pod update took 32.869789ms I0326 23:33:24.500276 3822 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-669c9bbc85-6sbq9] addLogicalPort annotation time took 32.89338ms I0326 23:33:24.500353 3822 pod.go:62] [openshift-insights/insights-runtime-extractor-59svj] pod update took 32.977811ms I0326 23:33:24.500375 3822 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-59svj] addLogicalPort annotation time took 33.006734ms I0326 23:33:24.500841 3822 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-hjsjl] creating logical port openshift-console_downloads-56d45c498b-hjsjl for pod on switch ip-10-0-0-231.ec2.internal I0326 23:33:24.500912 3822 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-hjsjl I0326 23:33:24.501450 3822 pods.go:271] [openshift-ingress/router-default-669c9bbc85-6sbq9] addLogicalPort took 34.216254ms, libovsdb time 834.123µs I0326 23:33:24.501717 3822 pods.go:271] [openshift-insights/insights-runtime-extractor-59svj] addLogicalPort took 34.471633ms, libovsdb time 1.057698ms I0326 23:33:24.522348 3822 pod.go:62] [openshift-console/downloads-56d45c498b-hjsjl] pod update took 21.456798ms I0326 23:33:24.522373 3822 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-hjsjl] addLogicalPort annotation time took 21.48479ms I0326 23:33:24.523366 3822 pods.go:271] [openshift-console/downloads-56d45c498b-hjsjl] addLogicalPort took 22.534367ms, libovsdb time 733.605µs I0326 23:33:25.133302 3822 cni.go:403] [openshift-insights/insights-runtime-extractor-59svj ecf1b1a813fab178809f04b20e8ee3ce7408ec82e0b19fcf7567279eb8ef304d network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-59svj ecf1b1a813fab178809f04b20e8ee3ce7408ec82e0b19fcf7567279eb8ef304d network default NAD default] I0326 23:33:25.136907 3822 cni.go:403] [openshift-ingress/router-default-669c9bbc85-6sbq9 d0e77235e0a4e12fc220dc1e71a1e88f101a268fe070d7125ce8f97136e7ba4d network default NAD default] ADD starting CNI request [openshift-ingress/router-default-669c9bbc85-6sbq9 d0e77235e0a4e12fc220dc1e71a1e88f101a268fe070d7125ce8f97136e7ba4d network default NAD default] I0326 23:33:25.142750 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-59svj, hostIfaceName: ecf1b1a813fab17, network: default, NAD default, SandboxID: "ecf1b1a813fab178809f04b20e8ee3ce7408ec82e0b19fcf7567279eb8ef304d", PCI device ID: , UID: "11e89576-fe51-4e97-810b-ec6e39c35732", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0326 23:33:25.145048 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-669c9bbc85-6sbq9, hostIfaceName: d0e77235e0a4e12, network: default, NAD default, SandboxID: "d0e77235e0a4e12fc220dc1e71a1e88f101a268fe070d7125ce8f97136e7ba4d", PCI device ID: , UID: "e1e0b200-af13-45e1-88f7-9fdb4b0c908b", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0326 23:33:25.163093 3822 cni.go:403] [openshift-console/downloads-56d45c498b-hjsjl 536e18559ad3516ecd5e432e8ed18574a1ea83045082ba069e74e080ed4ee3e2 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-hjsjl 536e18559ad3516ecd5e432e8ed18574a1ea83045082ba069e74e080ed4ee3e2 network default NAD default] I0326 23:33:25.172005 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-hjsjl, hostIfaceName: 536e18559ad3516, network: default, NAD default, SandboxID: "536e18559ad3516ecd5e432e8ed18574a1ea83045082ba069e74e080ed4ee3e2", PCI device ID: , UID: "0b5d4c04-e5ac-44ef-9344-81a25ab5fac0", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0326 23:33:25.220904 3822 cni.go:424] [openshift-console/downloads-56d45c498b-hjsjl 536e18559ad3516ecd5e432e8ed18574a1ea83045082ba069e74e080ed4ee3e2 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-hjsjl 536e18559ad3516ecd5e432e8ed18574a1ea83045082ba069e74e080ed4ee3e2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:4d:0f:7b:bd:c8\",\"name\":\"536e18559ad3516\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cb38c9fa-a0d8-404b-b68b-d46987a8517e\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:25.397986 3822 cni.go:424] [openshift-insights/insights-runtime-extractor-59svj ecf1b1a813fab178809f04b20e8ee3ce7408ec82e0b19fcf7567279eb8ef304d network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-59svj ecf1b1a813fab178809f04b20e8ee3ce7408ec82e0b19fcf7567279eb8ef304d network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:fa:22:6b:b1:f6\",\"name\":\"ecf1b1a813fab17\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/866b8b91-87c3-46dd-9828-d7607b0d1594\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:25.398895 3822 cni.go:424] [openshift-ingress/router-default-669c9bbc85-6sbq9 d0e77235e0a4e12fc220dc1e71a1e88f101a268fe070d7125ce8f97136e7ba4d network default NAD default] ADD finished CNI request [openshift-ingress/router-default-669c9bbc85-6sbq9 d0e77235e0a4e12fc220dc1e71a1e88f101a268fe070d7125ce8f97136e7ba4d network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:2f:2d:34:ce:96\",\"name\":\"d0e77235e0a4e12\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7fc3df81-2c3e-4fb8-ac34-dc9e231d2bc4\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:25.452797 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:33:25.452851 3822 node_tracker.go:235] Node ip-10-0-1-147.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-147.ec2.internal" I0326 23:33:25.452872 3822 node_tracker.go:169] Node ip-10-0-1-147.ec2.internal switch + router changed, syncing services in network "default" I0326 23:33:25.452797 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "ip-10-0-1-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:25.452889 3822 services_controller.go:574] Full service sync requested for network=default W0326 23:33:25.453251 3822 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:25.564155 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:33:25.564188 3822 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "ip-10-0-1-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:25.564217 3822 node_tracker.go:169] Node ip-10-0-1-147.ec2.internal switch + router changed, syncing services in network "default" I0326 23:33:25.564235 3822 services_controller.go:574] Full service sync requested for network=default W0326 23:33:25.564878 3822 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:25.565126 3822 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-147.ec2.internal for the network default I0326 23:33:25.566172 3822 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-147.ec2.internal" on network "default" took: 1.059038ms I0326 23:33:26.120596 3822 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:33:26.120629 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:33:26.120647 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 22.068µs I0326 23:33:26.120649 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 10.0.1.147 to PMTUD blocking rules I0326 23:33:26.120629 3822 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:33:26.120664 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:33:26.120670 3822 node_tracker.go:169] Node ip-10-0-1-147.ec2.internal switch + router changed, syncing services in network "default" I0326 23:33:26.120687 3822 services_controller.go:574] Full service sync requested for network=default W0326 23:33:26.121122 3822 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:26.127580 3822 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-147.ec2.internal. Updating no re-route policies I0326 23:33:26.140762 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 10.0.1.147 to PMTUD blocking rules I0326 23:33:26.140780 3822 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:33:26.206329 3822 base_network_controller_pods.go:487] [default/openshift-console/console-6969bb5f4f-kg498] creating logical port openshift-console_console-6969bb5f4f-kg498 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:33:26.206462 3822 kube.go:256] Updating pod openshift-console/console-6969bb5f4f-kg498 I0326 23:33:26.224494 3822 pod.go:62] [openshift-console/console-6969bb5f4f-kg498] pod update took 18.056262ms I0326 23:33:26.224523 3822 base_network_controller_pods.go:937] [default/openshift-console/console-6969bb5f4f-kg498] addLogicalPort annotation time took 18.090901ms I0326 23:33:26.225983 3822 pods.go:271] [openshift-console/console-6969bb5f4f-kg498] addLogicalPort took 19.669101ms, libovsdb time 1.050624ms W0326 23:33:26.234192 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:26.234207 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:26.234212 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:26.234320 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:33:26.234336 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default I0326 23:33:26.249657 3822 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:26.249687 3822 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:33:26.251115 3822 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5fd5865cf4-s6vd6, ips: 10.128.0.6 I0326 23:33:26.251143 3822 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:26.258199 3822 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv] creating logical port openshift-image-registry_image-registry-7d9b9d9ddb-cqfvv for pod on switch ip-10-0-0-231.ec2.internal I0326 23:33:26.258316 3822 kube.go:256] Updating pod openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv I0326 23:33:26.261226 3822 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:26.261289 3822 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:26.272743 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:33:26.272775 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:33:26.279133 3822 pod.go:62] [openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv] pod update took 20.873513ms I0326 23:33:26.279188 3822 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv] addLogicalPort annotation time took 20.932054ms I0326 23:33:26.280403 3822 pods.go:271] [openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv] addLogicalPort took 22.215975ms, libovsdb time 842.816µs I0326 23:33:26.568867 3822 cni.go:403] [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default] ADD starting CNI request [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default] I0326 23:33:26.577997 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6969bb5f4f-kg498, hostIfaceName: 85b2f135f459a47, network: default, NAD default, SandboxID: "85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635", PCI device ID: , UID: "36a3d25c-39c7-4251-b57e-2d20ae8ee588", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0326 23:33:26.621477 3822 cni.go:403] [openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv 4e93b172f734a8d9e4b1d6439c2187ae9023e7d63f37ce3607fa466ca8aeba20 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv 4e93b172f734a8d9e4b1d6439c2187ae9023e7d63f37ce3607fa466ca8aeba20 network default NAD default] I0326 23:33:26.625196 3822 cni.go:424] [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default] ADD finished CNI request [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:91:df:12:00:b1\",\"name\":\"85b2f135f459a47\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1eb15ce5-4250-4d0e-b02f-bb912f6c65aa\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:26.630095 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7d9b9d9ddb-cqfvv, hostIfaceName: 4e93b172f734a8d, network: default, NAD default, SandboxID: "4e93b172f734a8d9e4b1d6439c2187ae9023e7d63f37ce3607fa466ca8aeba20", PCI device ID: , UID: "b30cbc97-5652-403d-8101-4f1aca9be53b", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0326 23:33:26.673198 3822 cni.go:424] [openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv 4e93b172f734a8d9e4b1d6439c2187ae9023e7d63f37ce3607fa466ca8aeba20 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7d9b9d9ddb-cqfvv 4e93b172f734a8d9e4b1d6439c2187ae9023e7d63f37ce3607fa466ca8aeba20 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:b9:db:1e:a7:ec\",\"name\":\"4e93b172f734a8d\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4c98fdf-59d0-4424-9371-56c9006692f0\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:33:27.222517 3822 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:27.247654 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:27.247679 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:27.247689 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:27.250322 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:33:27.250350 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0326 23:33:27.391427 3822 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:33:29.595975 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:33:29.596016 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 48.268µs W0326 23:33:30.239894 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:33:30.711733 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:33:30.963191 3822 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:32.692130 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:33:33.552771 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:33:36.715682 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:36.715710 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:37.625059 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:37.625086 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:40.276144 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:33:40.307710 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0326 23:33:44.088881 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:44.088909 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:44.136648 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:44.136677 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:45.772569 3822 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:45.772665 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:45.772699 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:46.782346 3822 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-5c87bcb84c-2nqcd IP address from the namespace address-set, err: pod openshift-console/console-5c87bcb84c-2nqcd: no pod IPs found W0326 23:33:47.104864 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:47.104884 3822 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:47.104889 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:48.234515 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:48.781924 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:33:49.815596 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:49.815617 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:50.093240 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:33:50.813861 3822 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:50.813885 3822 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:50.813893 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:50.813901 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:54.237527 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:54.237551 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:54.572555 3822 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-68997f7bdb-wglmk IP address from the namespace address-set, err: pod openshift-console/console-68997f7bdb-wglmk: no pod IPs found W0326 23:33:54.585338 3822 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-68997f7bdb-msz5m IP address from the namespace address-set, err: pod openshift-console/console-68997f7bdb-msz5m: no pod IPs found I0326 23:33:55.329896 3822 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.330497 3822 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.330527 3822 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.330536 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0326 23:33:55.330548 3822 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.330553 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:33:56.145997 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.146016 3822 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.146021 3822 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.146026 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:33:56.364881 3822 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:56.364911 3822 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:56.364929 3822 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:56.364937 3822 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:58.719795 3822 cni.go:403] [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default] DEL starting CNI request [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default] I0326 23:33:58.790391 3822 cni.go:424] [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default] DEL finished CNI request [openshift-console/console-6969bb5f4f-kg498 85b2f135f459a4752aef3c51a56ef511422fb942b0674bf6740cf9f6b97cd635 network default NAD default], result "{}", err I0326 23:33:58.851023 3822 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-kg498 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:58.851057 3822 pods.go:176] Deleting pod: openshift-console/console-6969bb5f4f-kg498 I0326 23:33:58.852177 3822 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6969bb5f4f-kg498, ips: 10.128.0.20 I0326 23:33:58.852202 3822 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-kg498 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:58.861641 3822 base_network_controller_pods.go:487] [default/openshift-console/console-7b97c46849-jllrk] creating logical port openshift-console_console-7b97c46849-jllrk for pod on switch ip-10-0-0-231.ec2.internal I0326 23:33:58.861729 3822 kube.go:256] Updating pod openshift-console/console-7b97c46849-jllrk I0326 23:33:58.877040 3822 pod.go:62] [openshift-console/console-7b97c46849-jllrk] pod update took 15.339675ms I0326 23:33:58.877058 3822 base_network_controller_pods.go:937] [default/openshift-console/console-7b97c46849-jllrk] addLogicalPort annotation time took 15.361607ms I0326 23:33:58.878126 3822 pods.go:271] [openshift-console/console-7b97c46849-jllrk] addLogicalPort took 16.497934ms, libovsdb time 774.389µs I0326 23:33:59.226770 3822 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:33:59.263809 3822 cni.go:403] [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default] ADD starting CNI request [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default] I0326 23:33:59.265472 3822 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:33:59.271851 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7b97c46849-jllrk, hostIfaceName: 5279dee6858e3d7, network: default, NAD default, SandboxID: "5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7", PCI device ID: , UID: "0535aa1e-c0ad-4495-b6e4-29405e3f4af6", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0326 23:33:59.313596 3822 cni.go:424] [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default] ADD finished CNI request [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:0e:78:0e:c1:0e\",\"name\":\"5279dee6858e3d7\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4999a47e-8e3b-4776-a354-d43c45159e00\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:33:59.359010 3822 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-kg498 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:59.359034 3822 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-kg498 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:59.374840 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6969bb5f4f-kg498 I0326 23:33:59.374865 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6969bb5f4f-kg498 W0326 23:34:09.399095 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:34:09.463506 3822 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:09.464005 3822 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:09.905167 3822 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:09.905190 3822 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:09.911285 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6969bb5f4f-4kk7b I0326 23:34:09.911308 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6969bb5f4f-4kk7b I0326 23:34:13.617524 3822 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:13.618063 3822 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:34:13.945996 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0326 23:34:13.959912 3822 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:13.959930 3822 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:13.966206 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5fd5865cf4-sbv9c I0326 23:34:13.966231 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5fd5865cf4-sbv9c I0326 23:34:14.983874 3822 cni.go:403] [openshift-multus/network-metrics-daemon-nfmjv a4398d4f1e652f4bf9d4ba3cba6d7fe87f0f925d2206031b44f56d2fa4f08d3a network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-nfmjv a4398d4f1e652f4bf9d4ba3cba6d7fe87f0f925d2206031b44f56d2fa4f08d3a network default NAD default] I0326 23:34:14.991955 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-nfmjv, hostIfaceName: a4398d4f1e652f4, network: default, NAD default, SandboxID: "a4398d4f1e652f4bf9d4ba3cba6d7fe87f0f925d2206031b44f56d2fa4f08d3a", PCI device ID: , UID: "921e176c-8bf4-406d-9dda-67bbb8a95aff", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0326 23:34:15.033898 3822 cni.go:424] [openshift-multus/network-metrics-daemon-nfmjv a4398d4f1e652f4bf9d4ba3cba6d7fe87f0f925d2206031b44f56d2fa4f08d3a network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-nfmjv a4398d4f1e652f4bf9d4ba3cba6d7fe87f0f925d2206031b44f56d2fa4f08d3a network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:a2:a0:45:8c:7b\",\"name\":\"a4398d4f1e652f4\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af6250d6-01bb-4eb2-bd5b-f870c37b4ab3\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:34:17.432639 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:34:17.432645 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:34:17.432760 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:34:18.421642 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:34:18.421641 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:34:18.421808 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:34:18.443990 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:34:19.429771 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0326 23:34:34.723765 3822 obj_retry.go:464] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:34.724290 3822 obj_retry.go:464] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:35.314996 3822 obj_retry.go:458] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:35.315026 3822 obj_retry.go:458] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:35.321443 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5c87bcb84c-wj4vj I0326 23:34:35.321467 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5c87bcb84c-wj4vj I0326 23:35:27.521734 3822 cni.go:403] [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default] DEL starting CNI request [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default] I0326 23:35:27.596542 3822 cni.go:424] [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default] DEL finished CNI request [openshift-console/console-7b97c46849-jllrk 5279dee6858e3d7f986b8fa3578748459124ed5de8ae31116cf8e2f435e911a7 network default NAD default], result "{}", err I0326 23:35:27.640227 3822 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-jllrk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:27.640285 3822 pods.go:176] Deleting pod: openshift-console/console-7b97c46849-jllrk I0326 23:35:27.641673 3822 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7b97c46849-jllrk, ips: 10.128.0.22 I0326 23:35:27.641705 3822 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-jllrk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:27.652311 3822 base_network_controller_pods.go:487] [default/openshift-console/console-7479bb9c4d-nw2vr] creating logical port openshift-console_console-7479bb9c4d-nw2vr for pod on switch ip-10-0-0-231.ec2.internal I0326 23:35:27.652492 3822 kube.go:256] Updating pod openshift-console/console-7479bb9c4d-nw2vr I0326 23:35:27.670007 3822 pod.go:62] [openshift-console/console-7479bb9c4d-nw2vr] pod update took 17.54237ms I0326 23:35:27.670033 3822 base_network_controller_pods.go:937] [default/openshift-console/console-7479bb9c4d-nw2vr] addLogicalPort annotation time took 17.570847ms I0326 23:35:27.671085 3822 pods.go:271] [openshift-console/console-7479bb9c4d-nw2vr] addLogicalPort took 18.789004ms, libovsdb time 739.365µs I0326 23:35:27.961545 3822 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-jllrk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:27.961571 3822 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-jllrk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:27.967245 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b97c46849-jllrk I0326 23:35:27.967282 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b97c46849-jllrk I0326 23:35:28.006792 3822 cni.go:403] [openshift-console/console-7479bb9c4d-nw2vr 2c0f21f365031a2962209217daa0cb62061841ac6f97e699d5c47c0a4abda6fa network default NAD default] ADD starting CNI request [openshift-console/console-7479bb9c4d-nw2vr 2c0f21f365031a2962209217daa0cb62061841ac6f97e699d5c47c0a4abda6fa network default NAD default] I0326 23:35:28.014978 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7479bb9c4d-nw2vr, hostIfaceName: 2c0f21f365031a2, network: default, NAD default, SandboxID: "2c0f21f365031a2962209217daa0cb62061841ac6f97e699d5c47c0a4abda6fa", PCI device ID: , UID: "b1cb4ce2-cbf4-47a5-911a-f31648ba23fb", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0326 23:35:28.057323 3822 cni.go:424] [openshift-console/console-7479bb9c4d-nw2vr 2c0f21f365031a2962209217daa0cb62061841ac6f97e699d5c47c0a4abda6fa network default NAD default] ADD finished CNI request [openshift-console/console-7479bb9c4d-nw2vr 2c0f21f365031a2962209217daa0cb62061841ac6f97e699d5c47c0a4abda6fa network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:68:81:fb:f7:4b\",\"name\":\"2c0f21f365031a2\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9834e6bb-9d21-4e4b-a645-993abfb5315b\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:35:35.831971 3822 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0326 23:35:38.717940 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:35:38.825627 3822 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-gvrf5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:38.826186 3822 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-gvrf5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:39.310025 3822 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-gvrf5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:39.310070 3822 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-gvrf5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:39.316782 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b97c46849-gvrf5 I0326 23:35:39.316806 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b97c46849-gvrf5 I0326 23:35:42.727926 3822 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default] I0326 23:35:42.805823 3822 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 3d99a71112b59bd965161de70294693a6ab8a4dc49b6e5edd61ef79df1740f95 network default NAD default], result "{}", err I0326 23:35:42.844605 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:42.844637 3822 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 I0326 23:35:42.845872 3822 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9, ips: 10.128.0.9 I0326 23:35:42.845899 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:43.754546 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:43.754573 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:49.947028 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0326 23:38:09.935067 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0326 23:38:14.909115 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:38:43.941020 3822 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0326 23:38:48.828939 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=602 I0326 23:38:56.819000 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0326 23:39:00.311111 3822 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-s7jcd for pod on switch ip-10-0-0-231.ec2.internal I0326 23:39:00.311230 3822 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd I0326 23:39:00.335068 3822 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd] pod update took 23.866578ms I0326 23:39:00.335091 3822 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd] addLogicalPort annotation time took 23.893102ms I0326 23:39:00.336179 3822 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd] addLogicalPort took 25.081978ms, libovsdb time 797.374µs I0326 23:39:00.666651 3822 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd 8a35bf080f5c566a5820e2cfce2b3ddd5c1aeb3f6e2b265e0693583ab5bc7223 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd 8a35bf080f5c566a5820e2cfce2b3ddd5c1aeb3f6e2b265e0693583ab5bc7223 network default NAD default] I0326 23:39:00.674838 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-s7jcd, hostIfaceName: 8a35bf080f5c566, network: default, NAD default, SandboxID: "8a35bf080f5c566a5820e2cfce2b3ddd5c1aeb3f6e2b265e0693583ab5bc7223", PCI device ID: , UID: "7afe6d2d-d5cf-4915-976f-3d288163a292", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0326 23:39:00.921298 3822 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd 8a35bf080f5c566a5820e2cfce2b3ddd5c1aeb3f6e2b265e0693583ab5bc7223 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s7jcd 8a35bf080f5c566a5820e2cfce2b3ddd5c1aeb3f6e2b265e0693583ab5bc7223 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:7c:7f:db:c4:74\",\"name\":\"8a35bf080f5c566\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d79519ec-82da-429d-bd94-a9ed08783da2\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:39:03.413178 3822 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:39:10.825243 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=109 I0326 23:39:10.919903 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0326 23:39:12.925439 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 W0326 23:39:14.418583 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:39:14.452979 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:39:14.453050 3822 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-ch8fc IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-ch8fc: no pod IPs found I0326 23:39:15.824061 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0326 23:39:27.821050 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=78 I0326 23:39:45.819975 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0326 23:39:47.946158 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0326 23:40:00.182747 3822 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576140-89xn4 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:40:00.182852 3822 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 I0326 23:40:00.196740 3822 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4] pod update took 13.911568ms I0326 23:40:00.196762 3822 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4] addLogicalPort annotation time took 13.93817ms I0326 23:40:00.197855 3822 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4] addLogicalPort took 15.122815ms, libovsdb time 802.154µs I0326 23:40:00.551067 3822 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default] I0326 23:40:00.559056 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576140-89xn4, hostIfaceName: af49cd0402fe49f, network: default, NAD default, SandboxID: "af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d", PCI device ID: , UID: "59b66b9c-d16f-4faf-9bbf-e9032aa5abc7", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0326 23:40:00.806647 3822 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:0f:08:03:77:42\",\"name\":\"af49cd0402fe49f\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/209d299f-d143-4d7f-b1e4-97f8e8de18bd\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:40:03.628554 3822 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default] I0326 23:40:03.716627 3822 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 af49cd0402fe49fbd4d9b31ca30e66e3ce3c3eedff49abb7d602b8b13bd1049d network default NAD default], result "{}", err I0326 23:40:03.756691 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:03.756730 3822 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 I0326 23:40:03.757876 3822 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4, ips: 10.128.0.25 I0326 23:40:03.757899 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:04.643135 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:04.643168 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:48.856931 3822 namespace.go:98] [tsf] adding namespace I0326 23:40:48.857797 3822 namespace.go:102] [tsf] adding namespace took 839.207µs I0326 23:40:48.870816 3822 namespace.go:142] [tsf] updating namespace I0326 23:40:48.879117 3822 namespace.go:142] [tsf] updating namespace I0326 23:40:49.942845 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0326 23:40:52.859977 3822 namespace.go:98] [konflux-operator] adding namespace I0326 23:40:52.860047 3822 namespace.go:98] [tsf-tas] adding namespace I0326 23:40:52.860820 3822 namespace.go:102] [konflux-operator] adding namespace took 823.53µs I0326 23:40:52.861358 3822 namespace.go:102] [tsf-tas] adding namespace took 1.29525ms I0326 23:40:52.868059 3822 namespace.go:98] [cert-manager-operator] adding namespace I0326 23:40:52.868140 3822 namespace.go:98] [tsf-tpa] adding namespace I0326 23:40:52.868223 3822 namespace.go:98] [rhtpa-operator] adding namespace I0326 23:40:52.868349 3822 namespace.go:98] [tsf-keycloak] adding namespace I0326 23:40:52.868771 3822 namespace.go:102] [cert-manager-operator] adding namespace took 695.159µs I0326 23:40:52.869271 3822 namespace.go:102] [tsf-tpa] adding namespace took 1.10366ms I0326 23:40:52.869754 3822 namespace.go:102] [rhtpa-operator] adding namespace took 1.51624ms I0326 23:40:52.869776 3822 namespace.go:98] [rhbk-operator] adding namespace I0326 23:40:52.870188 3822 namespace.go:102] [rhbk-operator] adding namespace took 407.869µs I0326 23:40:52.870579 3822 namespace.go:102] [tsf-keycloak] adding namespace took 2.217181ms I0326 23:40:52.870607 3822 namespace.go:98] [konflux-ui] adding namespace I0326 23:40:52.871027 3822 namespace.go:102] [konflux-ui] adding namespace took 412.878µs I0326 23:40:52.877167 3822 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:52.896368 3822 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:52.902520 3822 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:52.918092 3822 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:52.926058 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:52.926091 3822 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:52.926207 3822 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:52.946908 3822 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:52.951161 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:52.951184 3822 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:52.955229 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:52.955244 3822 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:52.955420 3822 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:52.958024 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:52.972980 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:52.986323 3822 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:53.001659 3822 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:53.016085 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:53.035175 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:53.035244 3822 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:53.061940 3822 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:53.072401 3822 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:53.073970 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:53.088961 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:53.910793 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:53.923308 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:00.011788 3822 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012240 3822 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012284 3822 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012297 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:00.012308 3822 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012312 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:01.022292 3822 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022844 3822 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022873 3822 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022882 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:01.022900 3822 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022910 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:01.037890 3822 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:01.037910 3822 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:01.037921 3822 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:01.037933 3822 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048666 3822 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048693 3822 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048703 3822 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048713 3822 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:06.077223 3822 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077723 3822 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077749 3822 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077757 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:06.077771 3822 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077777 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:07.105476 3822 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.105501 3822 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.105516 3822 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.105528 3822 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.841799 3822 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-6mqkb for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:07.841915 3822 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb I0326 23:41:07.869456 3822 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb] pod update took 27.572809ms I0326 23:41:07.869478 3822 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb] addLogicalPort annotation time took 27.599256ms I0326 23:41:07.870619 3822 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb] addLogicalPort took 28.841917ms, libovsdb time 834.705µs I0326 23:41:08.171813 3822 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.172253 3822 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.172299 3822 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.172308 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:08.172323 3822 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.172331 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:08.223189 3822 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.223641 3822 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.223665 3822 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.223673 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:08.223709 3822 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.223720 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:08.499533 3822 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb 6389787ede27d8e1bf608a8bb97d8c878cc916d658287fd270847e346173c08b network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb 6389787ede27d8e1bf608a8bb97d8c878cc916d658287fd270847e346173c08b network default NAD default] I0326 23:41:08.508809 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-6mqkb, hostIfaceName: 6389787ede27d8e, network: default, NAD default, SandboxID: "6389787ede27d8e1bf608a8bb97d8c878cc916d658287fd270847e346173c08b", PCI device ID: , UID: "3e30b878-7338-4e45-85a2-805bef69dd2e", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0326 23:41:08.552398 3822 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb 6389787ede27d8e1bf608a8bb97d8c878cc916d658287fd270847e346173c08b network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-6mqkb 6389787ede27d8e1bf608a8bb97d8c878cc916d658287fd270847e346173c08b network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:da:1c:d2:1c:ae\",\"name\":\"6389787ede27d8e\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/93bb8b14-aaa8-4565-97b2-b584b6c893a6\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:09.201335 3822 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.201367 3822 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.201383 3822 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.201393 3822 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255351 3822 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255367 3822 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255376 3822 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255386 3822 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:41:11.883011 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:41:12.911508 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0326 23:41:12.914215 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 W0326 23:41:13.422175 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0326 23:41:13.432803 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:41:17.754824 3822 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-98dz2 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:17.754983 3822 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 I0326 23:41:17.777145 3822 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2] pod update took 22.189175ms I0326 23:41:17.777170 3822 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2] addLogicalPort annotation time took 22.22021ms I0326 23:41:17.778414 3822 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2] addLogicalPort took 23.609834ms, libovsdb time 902.166µs I0326 23:41:18.112209 3822 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default] I0326 23:41:18.120817 3822 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-98dz2, hostIfaceName: d41c78de478a8c3, network: default, NAD default, SandboxID: "d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119", PCI device ID: , UID: "678b1d64-073a-4815-b07e-ca2c094a4fc9", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0326 23:41:18.163455 3822 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:2a:e1:8e:97:80\",\"name\":\"d41c78de478a8c3\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/06e10584-78dd-4587-aa76-323f180b2661\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:19.147302 3822 namespace.go:142] [openshift-operators] updating namespace I0326 23:41:19.158554 3822 namespace.go:142] [openshift-operators] updating namespace W0326 23:41:20.959715 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.147296 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.187977 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.219704 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.254200 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:41:22.518253 3822 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.518852 3822 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.518878 3822 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.518885 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:22.518897 3822 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.518902 3822 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... W0326 23:41:22.915048 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:41:23.602362 3822 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:23.602389 3822 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:23.602405 3822 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:23.602417 3822 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:26.365165 3822 namespace.go:142] [openshift-operators] updating namespace I0326 23:41:26.794118 3822 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-fdq8g for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:26.794222 3822 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g I0326 23:41:26.817832 3822 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g] pod update took 23.632077ms I0326 23:41:26.817857 3822 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g] addLogicalPort annotation time took 23.659298ms I0326 23:41:26.819002 3822 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g] addLogicalPort took 24.899408ms, libovsdb time 856.106µs I0326 23:41:27.181806 3822 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g d56fc31bf9744b64b3432ec58969c0e12fef0941c030da42eaa6e2a1e80e7440 network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g d56fc31bf9744b64b3432ec58969c0e12fef0941c030da42eaa6e2a1e80e7440 network default NAD default] I0326 23:41:27.190449 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-fdq8g, hostIfaceName: d56fc31bf9744b6, network: default, NAD default, SandboxID: "d56fc31bf9744b64b3432ec58969c0e12fef0941c030da42eaa6e2a1e80e7440", PCI device ID: , UID: "1a40bde8-1059-472a-b7d7-8b851e3c72c2", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0326 23:41:27.232129 3822 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g d56fc31bf9744b64b3432ec58969c0e12fef0941c030da42eaa6e2a1e80e7440 network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-fdq8g d56fc31bf9744b64b3432ec58969c0e12fef0941c030da42eaa6e2a1e80e7440 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:e8:6b:d8:38:51\",\"name\":\"d56fc31bf9744b6\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5aabb3ca-39f7-49b0-8516-9237455aaa31\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:31.915312 3822 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-27pgw for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:31.915445 3822 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw I0326 23:41:32.001290 3822 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw W0326 23:41:32.036792 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:41:32.072601 3822 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw] pod update took 157.184888ms I0326 23:41:32.072621 3822 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw] addLogicalPort annotation time took 157.20916ms I0326 23:41:32.073650 3822 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw] addLogicalPort took 158.354658ms, libovsdb time 737.108µs I0326 23:41:32.267525 3822 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw 500582f5e57b928abe60572379695277f9b354a4d9bd11c41b0f28a160fdd8ac network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw 500582f5e57b928abe60572379695277f9b354a4d9bd11c41b0f28a160fdd8ac network default NAD default] I0326 23:41:32.275835 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-27pgw, hostIfaceName: 500582f5e57b928, network: default, NAD default, SandboxID: "500582f5e57b928abe60572379695277f9b354a4d9bd11c41b0f28a160fdd8ac", PCI device ID: , UID: "f5eebc7b-1e9b-42d1-989c-c12edf818253", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0326 23:41:32.525256 3822 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw 500582f5e57b928abe60572379695277f9b354a4d9bd11c41b0f28a160fdd8ac network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-27pgw 500582f5e57b928abe60572379695277f9b354a4d9bd11c41b0f28a160fdd8ac network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:ec:94:29:ae:00\",\"name\":\"500582f5e57b928\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae6da454-3f2b-4386-9d28-721aa42fda78\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:35.470497 3822 namespace.go:98] [trusted-artifact-signer] adding namespace I0326 23:41:35.471476 3822 namespace.go:102] [trusted-artifact-signer] adding namespace took 956.354µs I0326 23:41:35.531342 3822 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:35.544100 3822 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:35.760314 3822 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-w7scq] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-w7scq for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:35.760441 3822 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-w7scq I0326 23:41:35.830850 3822 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-w7scq I0326 23:41:35.864983 3822 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-w7scq] pod update took 104.562679ms I0326 23:41:35.865003 3822 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-w7scq] addLogicalPort annotation time took 104.587451ms I0326 23:41:35.866184 3822 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-w7scq] addLogicalPort took 105.883601ms, libovsdb time 805.294µs W0326 23:41:36.076766 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:41:36.115536 3822 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-w7scq 85dc10a1d0e3158c7ed43a75afcf558ef31b02b94184915765f8c2d4dce87fcc network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-w7scq 85dc10a1d0e3158c7ed43a75afcf558ef31b02b94184915765f8c2d4dce87fcc network default NAD default] I0326 23:41:36.123929 3822 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-w7scq, hostIfaceName: 85dc10a1d0e3158, network: default, NAD default, SandboxID: "85dc10a1d0e3158c7ed43a75afcf558ef31b02b94184915765f8c2d4dce87fcc", PCI device ID: , UID: "3b95870a-8c3f-4e01-a7e2-c9b7f9668d58", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0326 23:41:36.165230 3822 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-w7scq 85dc10a1d0e3158c7ed43a75afcf558ef31b02b94184915765f8c2d4dce87fcc network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-w7scq 85dc10a1d0e3158c7ed43a75afcf558ef31b02b94184915765f8c2d4dce87fcc network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:3c:10:b7:ef:0d\",\"name\":\"85dc10a1d0e3158\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6423826d-82b0-43e4-8ec4-29e6bd2b942b\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:37.406668 3822 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-mwpwt for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:37.406814 3822 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt I0326 23:41:37.429821 3822 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt] pod update took 23.042306ms I0326 23:41:37.429848 3822 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt] addLogicalPort annotation time took 23.074017ms I0326 23:41:37.431304 3822 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt] addLogicalPort took 24.654587ms, libovsdb time 1.031684ms I0326 23:41:38.157208 3822 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt f8c9fe033f23a2056832ae791511b45fff94663b713ad536dd28d054ea2bf8b2 network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt f8c9fe033f23a2056832ae791511b45fff94663b713ad536dd28d054ea2bf8b2 network default NAD default] I0326 23:41:38.165887 3822 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-mwpwt, hostIfaceName: f8c9fe033f23a20, network: default, NAD default, SandboxID: "f8c9fe033f23a2056832ae791511b45fff94663b713ad536dd28d054ea2bf8b2", PCI device ID: , UID: "7be43450-b99b-4d2d-abf0-c7fa74b8ffcd", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0326 23:41:38.414279 3822 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt f8c9fe033f23a2056832ae791511b45fff94663b713ad536dd28d054ea2bf8b2 network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-mwpwt f8c9fe033f23a2056832ae791511b45fff94663b713ad536dd28d054ea2bf8b2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:27:f6:35:d5:ea\",\"name\":\"f8c9fe033f23a20\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c8305dde-9a6e-40dc-b56d-b40eb31f06c0\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:41.328368 3822 namespace.go:98] [openshift-pipelines] adding namespace I0326 23:41:41.329243 3822 namespace.go:102] [openshift-pipelines] adding namespace took 849.636µs I0326 23:41:41.348188 3822 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:41:42.089012 3822 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:42.160579 3822 namespace.go:98] [cert-manager] adding namespace I0326 23:41:42.161532 3822 namespace.go:102] [cert-manager] adding namespace took 928.928µs I0326 23:41:42.180920 3822 namespace.go:142] [cert-manager] updating namespace I0326 23:41:42.211093 3822 namespace.go:142] [cert-manager] updating namespace I0326 23:41:42.415675 3822 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:42.631123 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:42.817333 3822 namespace.go:142] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] updating namespace I0326 23:41:43.207787 3822 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:43.409111 3822 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:43.605379 3822 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:43.802992 3822 namespace.go:142] [tsf] updating namespace I0326 23:41:43.997122 3822 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-4cslk] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-4cslk for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:43.997214 3822 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-4cslk I0326 23:41:44.023393 3822 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-4cslk] pod update took 26.201651ms I0326 23:41:44.023414 3822 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-4cslk] addLogicalPort annotation time took 26.229487ms I0326 23:41:44.024537 3822 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-4cslk] addLogicalPort took 27.436583ms, libovsdb time 793.473µs I0326 23:41:44.042057 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:44.221999 3822 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:44.357889 3822 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-4cslk e92974b6f256f4d1104d524b0ff8757ed6bf4f762ae5c431e854b1cae3e231bc network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-4cslk e92974b6f256f4d1104d524b0ff8757ed6bf4f762ae5c431e854b1cae3e231bc network default NAD default] I0326 23:41:44.366609 3822 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-4cslk, hostIfaceName: e92974b6f256f4d, network: default, NAD default, SandboxID: "e92974b6f256f4d1104d524b0ff8757ed6bf4f762ae5c431e854b1cae3e231bc", PCI device ID: , UID: "6702de59-28ab-40c8-a906-6ea46dc36957", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0326 23:41:44.412137 3822 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-4cslk e92974b6f256f4d1104d524b0ff8757ed6bf4f762ae5c431e854b1cae3e231bc network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-4cslk e92974b6f256f4d1104d524b0ff8757ed6bf4f762ae5c431e854b1cae3e231bc network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:11:a5:b1:7b:65\",\"name\":\"e92974b6f256f4d\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb4766b7-f834-4349-9f53-b082ef60b7de\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:44.455284 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:44.551708 3822 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:44.561367 3822 namespace.go:142] [default] updating namespace I0326 23:41:44.577247 3822 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:44.583431 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:44.591931 3822 namespace.go:142] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] updating namespace I0326 23:41:44.601243 3822 namespace.go:142] [openshift] updating namespace I0326 23:41:44.625014 3822 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:44.636490 3822 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:44.642669 3822 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:44.655565 3822 namespace.go:142] [tsf] updating namespace I0326 23:41:44.666131 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:44.678020 3822 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:44.691977 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:44.799453 3822 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:44.898880 3822 namespace.go:142] [default] updating namespace I0326 23:41:45.028920 3822 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:45.110012 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:45.195124 3822 namespace.go:142] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] updating namespace I0326 23:41:45.292381 3822 namespace.go:142] [openshift] updating namespace I0326 23:41:45.395751 3822 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:45.498349 3822 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:45.607878 3822 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:45.700639 3822 namespace.go:142] [tsf] updating namespace I0326 23:41:45.790942 3822 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:45.896096 3822 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:45.991521 3822 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:46.412777 3822 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:41:46.918094 3822 namespace.go:142] [cert-manager] updating namespace W0326 23:41:47.059069 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:41:47.183067 3822 namespace.go:142] [cert-manager] updating namespace I0326 23:41:47.297058 3822 namespace.go:142] [cert-manager] updating namespace W0326 23:41:48.133058 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:41:48.133088 3822 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:41:49.052565 3822 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:49.053109 3822 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:49.091435 3822 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:49.091455 3822 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:49.106418 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0326 23:41:49.106438 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0326 23:41:53.946666 3822 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-gd69k for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:53.946787 3822 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k I0326 23:41:53.963332 3822 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k] pod update took 16.579823ms I0326 23:41:53.963359 3822 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k] addLogicalPort annotation time took 16.616636ms I0326 23:41:53.964711 3822 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k] addLogicalPort took 18.064089ms, libovsdb time 851.721µs I0326 23:41:54.617055 3822 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-w6mls for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:54.617135 3822 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls I0326 23:41:54.638096 3822 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls] pod update took 20.983926ms I0326 23:41:54.638116 3822 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls] addLogicalPort annotation time took 21.008584ms I0326 23:41:54.639176 3822 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls] addLogicalPort took 22.134002ms, libovsdb time 716.685µs I0326 23:41:54.723153 3822 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-5bjt5 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:41:54.723239 3822 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5 I0326 23:41:54.742111 3822 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5] pod update took 18.895196ms I0326 23:41:54.742139 3822 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5] addLogicalPort annotation time took 18.925508ms I0326 23:41:54.743553 3822 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5] addLogicalPort took 20.412535ms, libovsdb time 919.455µs I0326 23:41:54.983306 3822 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls c3994f12b41a50762c3db96fe7065fd53515a4f412d2eca6f531a366808e9ea8 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls c3994f12b41a50762c3db96fe7065fd53515a4f412d2eca6f531a366808e9ea8 network default NAD default] I0326 23:41:54.991741 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-w6mls, hostIfaceName: c3994f12b41a507, network: default, NAD default, SandboxID: "c3994f12b41a50762c3db96fe7065fd53515a4f412d2eca6f531a366808e9ea8", PCI device ID: , UID: "7d550246-4666-4c07-b408-ea7cecebe1dd", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0326 23:41:55.082214 3822 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5 812731c72713851d5e3ed23900fc197ea59836b896ed082be50db7dcbfd4d0c0 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5 812731c72713851d5e3ed23900fc197ea59836b896ed082be50db7dcbfd4d0c0 network default NAD default] I0326 23:41:55.090813 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-5bjt5, hostIfaceName: 812731c72713851, network: default, NAD default, SandboxID: "812731c72713851d5e3ed23900fc197ea59836b896ed082be50db7dcbfd4d0c0", PCI device ID: , UID: "b7d911e2-03a7-4dd0-aed7-d079f7451bb3", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0326 23:41:55.243347 3822 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls c3994f12b41a50762c3db96fe7065fd53515a4f412d2eca6f531a366808e9ea8 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w6mls c3994f12b41a50762c3db96fe7065fd53515a4f412d2eca6f531a366808e9ea8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:b8:09:e1:f8:a0\",\"name\":\"c3994f12b41a507\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da6448dc-d741-441a-bef3-aa0b58957c76\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:41:55.341721 3822 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5 812731c72713851d5e3ed23900fc197ea59836b896ed082be50db7dcbfd4d0c0 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-5bjt5 812731c72713851d5e3ed23900fc197ea59836b896ed082be50db7dcbfd4d0c0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:99:2b:5e:30:69\",\"name\":\"812731c72713851\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/018d4dca-24c3-4e07-a2f3-9ca984a0f1e0\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:41:58.272059 3822 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:41:58.272080 3822 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:41:58.272085 3822 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:41:58.291857 3822 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:41:58.291874 3822 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:41:58.291879 3822 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:41:59.053747 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:41:59.053770 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:41:59.110238 3822 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:41:59.110270 3822 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:41:59.110281 3822 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:41:59.110287 3822 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:41:59.711737 3822 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k 24854813bf7206003310bf6189c4dd446e8fea75f3b8587ded3a16c444cc5fd6 network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k 24854813bf7206003310bf6189c4dd446e8fea75f3b8587ded3a16c444cc5fd6 network default NAD default] I0326 23:41:59.720694 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-gd69k, hostIfaceName: 24854813bf72060, network: default, NAD default, SandboxID: "24854813bf7206003310bf6189c4dd446e8fea75f3b8587ded3a16c444cc5fd6", PCI device ID: , UID: "7491d261-ef5b-49ad-b4bc-439533e7a555", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0326 23:41:59.766521 3822 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k 24854813bf7206003310bf6189c4dd446e8fea75f3b8587ded3a16c444cc5fd6 network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-gd69k 24854813bf7206003310bf6189c4dd446e8fea75f3b8587ded3a16c444cc5fd6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:b6:35:34:58:4f\",\"name\":\"24854813bf72060\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eda518d4-0fc7-4758-abb2-c526fcada5f0\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:42:00.991206 3822 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0326 23:42:05.139760 3822 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0326 23:42:10.120868 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:10.120894 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:10.120900 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:10.120905 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:42:12.544393 3822 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:42:12.544405 3822 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:42:12.544440 3822 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 38.47µs W0326 23:42:20.085509 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0326 23:42:26.144707 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0326 23:42:29.245970 3822 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:29.246522 3822 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:29.266926 3822 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:29.266943 3822 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:29.274150 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0326 23:42:29.274172 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0326 23:42:36.153318 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:42:36.153338 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:42:36.155369 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:42:37.404435 3822 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:42:51.141070 3822 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-nc8x6 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:42:51.141174 3822 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6 I0326 23:42:51.156531 3822 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6] pod update took 15.386539ms I0326 23:42:51.156559 3822 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6] addLogicalPort annotation time took 15.418756ms I0326 23:42:51.157856 3822 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6] addLogicalPort took 16.79718ms, libovsdb time 871.129µs I0326 23:42:52.098958 3822 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6 099dbb54bdb9058a675b87bf7fe4b1f472e2aaf30d05cbca4f0360571e05689e network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6 099dbb54bdb9058a675b87bf7fe4b1f472e2aaf30d05cbca4f0360571e05689e network default NAD default] I0326 23:42:52.107073 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-nc8x6, hostIfaceName: 099dbb54bdb9058, network: default, NAD default, SandboxID: "099dbb54bdb9058a675b87bf7fe4b1f472e2aaf30d05cbca4f0360571e05689e", PCI device ID: , UID: "3f48f5b5-44e4-46a3-ad21-7ab3ae4923b2", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0326 23:42:52.357965 3822 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6 099dbb54bdb9058a675b87bf7fe4b1f472e2aaf30d05cbca4f0360571e05689e network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-nc8x6 099dbb54bdb9058a675b87bf7fe4b1f472e2aaf30d05cbca4f0360571e05689e network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:2e:a1:d6:48:1c\",\"name\":\"099dbb54bdb9058\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b34c093d-92c3-4d05-a74c-55443c881f46\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:42:55.626181 3822 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:42:55.626204 3822 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:42:55.626210 3822 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0326 23:42:57.518471 3822 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:57.519035 3822 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:58.260598 3822 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:58.260628 3822 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:58.268071 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:42:58.268096 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0326 23:43:00.285984 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:00.286003 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:00.286025 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:43:08.740553 3822 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-9f6kx] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-9f6kx for pod on switch ip-10-0-0-231.ec2.internal I0326 23:43:08.740688 3822 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-9f6kx I0326 23:43:08.756137 3822 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-9f6kx] pod update took 15.478536ms I0326 23:43:08.756159 3822 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-9f6kx] addLogicalPort annotation time took 15.506116ms I0326 23:43:08.757438 3822 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-9f6kx] addLogicalPort took 16.89995ms, libovsdb time 862.869µs I0326 23:43:09.095200 3822 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-9f6kx 921852f636926b64f8b2b8bbe855d3c28a4ba05fa5a35fd4ece0f7d1a6059263 network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-9f6kx 921852f636926b64f8b2b8bbe855d3c28a4ba05fa5a35fd4ece0f7d1a6059263 network default NAD default] I0326 23:43:09.104168 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-9f6kx, hostIfaceName: 921852f636926b6, network: default, NAD default, SandboxID: "921852f636926b64f8b2b8bbe855d3c28a4ba05fa5a35fd4ece0f7d1a6059263", PCI device ID: , UID: "f6aa2de8-a722-4067-9823-698190141df3", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0326 23:43:09.354992 3822 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-9f6kx 921852f636926b64f8b2b8bbe855d3c28a4ba05fa5a35fd4ece0f7d1a6059263 network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-9f6kx 921852f636926b64f8b2b8bbe855d3c28a4ba05fa5a35fd4ece0f7d1a6059263 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:cd:d4:39:dc:ee\",\"name\":\"921852f636926b6\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/98d6b59e-9c17-44bf-b799-1f807f2f5028\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:43:25.671869 3822 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-dd2jc for pod on switch ip-10-0-0-231.ec2.internal I0326 23:43:25.671960 3822 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc I0326 23:43:25.684921 3822 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc] pod update took 12.978462ms I0326 23:43:25.684947 3822 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc] addLogicalPort annotation time took 13.009769ms I0326 23:43:25.686171 3822 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc] addLogicalPort took 14.321422ms, libovsdb time 854.207µs I0326 23:43:26.037979 3822 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc 5f93d31ebabc51d524d706cd528aab49eaa7083d94fb9142c6571d186e0e4ec3 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc 5f93d31ebabc51d524d706cd528aab49eaa7083d94fb9142c6571d186e0e4ec3 network default NAD default] I0326 23:43:26.046344 3822 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-dd2jc, hostIfaceName: 5f93d31ebabc51d, network: default, NAD default, SandboxID: "5f93d31ebabc51d524d706cd528aab49eaa7083d94fb9142c6571d186e0e4ec3", PCI device ID: , UID: "75d07de2-ab36-4814-82c1-5fdc82d4e2df", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0326 23:43:26.296808 3822 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc 5f93d31ebabc51d524d706cd528aab49eaa7083d94fb9142c6571d186e0e4ec3 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-dd2jc 5f93d31ebabc51d524d706cd528aab49eaa7083d94fb9142c6571d186e0e4ec3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:06:15:34:49:bb\",\"name\":\"5f93d31ebabc51d\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b227c0dd-9b22-4a4c-b052-ffd09ecd0804\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:43:28.793223 3822 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0326 23:43:31.643015 3822 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:31.643589 3822 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:38.637281 3822 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:38.637863 3822 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:39.664744 3822 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:39.664767 3822 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:40.016385 3822 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:40.016905 3822 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:40.519752 3822 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:40.519777 3822 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:40.527951 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:43:40.527974 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0326 23:43:42.544363 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:43:42.548520 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:42.548535 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:44:00.944725 3822 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0326 23:44:20.938705 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0326 23:44:29.697930 3822 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:44:31.791338 3822 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:31.791361 3822 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:31.798192 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:44:31.798215 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:44:40.979171 3822 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:44:40.979753 3822 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:44:41.002975 3822 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:41.002994 3822 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:41.010162 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:44:41.010182 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:44:43.694247 3822 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-7bb776885d-kb6q7] creating logical port tsf-tas_fulcio-server-7bb776885d-kb6q7 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:44:43.694375 3822 kube.go:256] Updating pod tsf-tas/fulcio-server-7bb776885d-kb6q7 I0326 23:44:43.712599 3822 pod.go:62] [tsf-tas/fulcio-server-7bb776885d-kb6q7] pod update took 18.243461ms I0326 23:44:43.712629 3822 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-7bb776885d-kb6q7] addLogicalPort annotation time took 18.276693ms I0326 23:44:43.713993 3822 pods.go:271] [tsf-tas/fulcio-server-7bb776885d-kb6q7] addLogicalPort took 19.763653ms, libovsdb time 838.96µs I0326 23:44:44.051424 3822 cni.go:403] [tsf-tas/fulcio-server-7bb776885d-kb6q7 91a5dce5fa59dca71d7c05f8076c5fb8c4bb0e89976f89a15df2fad272af4d70 network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-7bb776885d-kb6q7 91a5dce5fa59dca71d7c05f8076c5fb8c4bb0e89976f89a15df2fad272af4d70 network default NAD default] I0326 23:44:44.060067 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-7bb776885d-kb6q7, hostIfaceName: 91a5dce5fa59dca, network: default, NAD default, SandboxID: "91a5dce5fa59dca71d7c05f8076c5fb8c4bb0e89976f89a15df2fad272af4d70", PCI device ID: , UID: "8434c37f-2667-42c8-924e-c518ddd76fba", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0326 23:44:44.131381 3822 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-wddsf-zzkhg] creating logical port tsf-tas_rekor-createtree-job-wddsf-zzkhg for pod on switch ip-10-0-0-231.ec2.internal I0326 23:44:44.131499 3822 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-wddsf-zzkhg I0326 23:44:44.150252 3822 pod.go:62] [tsf-tas/rekor-createtree-job-wddsf-zzkhg] pod update took 18.769744ms I0326 23:44:44.150290 3822 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-wddsf-zzkhg] addLogicalPort annotation time took 18.813877ms I0326 23:44:44.151503 3822 pods.go:271] [tsf-tas/rekor-createtree-job-wddsf-zzkhg] addLogicalPort took 20.138931ms, libovsdb time 843.784µs I0326 23:44:44.310734 3822 cni.go:424] [tsf-tas/fulcio-server-7bb776885d-kb6q7 91a5dce5fa59dca71d7c05f8076c5fb8c4bb0e89976f89a15df2fad272af4d70 network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-7bb776885d-kb6q7 91a5dce5fa59dca71d7c05f8076c5fb8c4bb0e89976f89a15df2fad272af4d70 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:0f:4e:e5:51:c9\",\"name\":\"91a5dce5fa59dca\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5d6730c-43e6-4d06-8ca9-cac58035d3db\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:44:44.482088 3822 cni.go:403] [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default] I0326 23:44:44.489938 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-wddsf-zzkhg, hostIfaceName: f671fd63f9e9a2a, network: default, NAD default, SandboxID: "f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c", PCI device ID: , UID: "b113fe0a-bd4b-4a3b-9ab2-6291a17cb3bf", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0326 23:44:44.533812 3822 cni.go:424] [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:75:11:2a:61:06\",\"name\":\"f671fd63f9e9a2a\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49d0f92f-5ced-416e-bf7f-a705ee7e0c2e\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:44:46.634429 3822 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-6f56d4ff5c-z9qj7] creating logical port tsf-tas_trillian-db-6f56d4ff5c-z9qj7 for pod on switch ip-10-0-0-231.ec2.internal I0326 23:44:46.634520 3822 kube.go:256] Updating pod tsf-tas/trillian-db-6f56d4ff5c-z9qj7 I0326 23:44:46.654347 3822 pod.go:62] [tsf-tas/trillian-db-6f56d4ff5c-z9qj7] pod update took 19.851165ms I0326 23:44:46.654368 3822 base_network_controller_pods.go:937] [default/tsf-tas/trillian-db-6f56d4ff5c-z9qj7] addLogicalPort annotation time took 19.877042ms I0326 23:44:46.655600 3822 pods.go:271] [tsf-tas/trillian-db-6f56d4ff5c-z9qj7] addLogicalPort took 21.186122ms, libovsdb time 868.733µs W0326 23:44:46.892515 3822 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:46.892539 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:46.913620 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:46.913638 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0326 23:44:48.829449 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=108 W0326 23:44:51.261091 3822 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0326 23:44:52.391109 3822 cni.go:403] [tsf-tas/trillian-db-6f56d4ff5c-z9qj7 f28cd76fe8ac587fbfbc22b2e04990a0b35bf1c3fb0352044611ef6504ac6696 network default NAD default] ADD starting CNI request [tsf-tas/trillian-db-6f56d4ff5c-z9qj7 f28cd76fe8ac587fbfbc22b2e04990a0b35bf1c3fb0352044611ef6504ac6696 network default NAD default] I0326 23:44:52.399299 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-6f56d4ff5c-z9qj7, hostIfaceName: f28cd76fe8ac587, network: default, NAD default, SandboxID: "f28cd76fe8ac587fbfbc22b2e04990a0b35bf1c3fb0352044611ef6504ac6696", PCI device ID: , UID: "e6521c3b-18ca-4997-baef-ac5c20f4564e", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0326 23:44:52.649563 3822 cni.go:424] [tsf-tas/trillian-db-6f56d4ff5c-z9qj7 f28cd76fe8ac587fbfbc22b2e04990a0b35bf1c3fb0352044611ef6504ac6696 network default NAD default] ADD finished CNI request [tsf-tas/trillian-db-6f56d4ff5c-z9qj7 f28cd76fe8ac587fbfbc22b2e04990a0b35bf1c3fb0352044611ef6504ac6696 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:c0:d3:9c:7f:00\",\"name\":\"f28cd76fe8ac587\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2fd2398-bc85-429e-b544-10db0cb3ed0c\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:44:53.950421 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:44:54.825463 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=47 W0326 23:44:55.943409 3822 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0326 23:44:56.994555 3822 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:44:57.913214 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0326 23:45:00.128517 3822 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:00.129075 3822 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:01.148168 3822 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:01.148191 3822 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:45:06.984335 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0326 23:45:31.274871 3822 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:31.275434 3822 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:31.300067 3822 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:31.300084 3822 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:31.307236 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0326 23:45:31.307278 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0326 23:45:32.116448 3822 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-vbftd] creating logical port tsf_patch-tekton-config-vbftd for pod on switch ip-10-0-0-231.ec2.internal I0326 23:45:32.116562 3822 kube.go:256] Updating pod tsf/patch-tekton-config-vbftd I0326 23:45:32.131310 3822 pod.go:62] [tsf/patch-tekton-config-vbftd] pod update took 14.7692ms I0326 23:45:32.131330 3822 base_network_controller_pods.go:937] [default/tsf/patch-tekton-config-vbftd] addLogicalPort annotation time took 14.817066ms I0326 23:45:32.132638 3822 pods.go:271] [tsf/patch-tekton-config-vbftd] addLogicalPort took 16.205142ms, libovsdb time 839.685µs I0326 23:45:32.474027 3822 cni.go:403] [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default] ADD starting CNI request [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default] I0326 23:45:32.482085 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-vbftd, hostIfaceName: 41f11c5013bbd67, network: default, NAD default, SandboxID: "41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4", PCI device ID: , UID: "2ee04082-155b-45bd-9c66-6aa1a2ae3936", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0326 23:45:32.734248 3822 cni.go:424] [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default] ADD finished CNI request [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:43:8b:f1:85:31\",\"name\":\"41f11c5013bbd67\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c14f0cf3-8a71-488f-be7e-a324adb4d5cd\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:45:40.156601 3822 cni.go:403] [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default] DEL starting CNI request [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default] I0326 23:45:40.233740 3822 cni.go:424] [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default] DEL finished CNI request [tsf/patch-tekton-config-vbftd 41f11c5013bbd67e399e674410f646cdea3af37a311f626dcea07403be227fb4 network default NAD default], result "{}", err I0326 23:45:40.289682 3822 obj_retry.go:464] Detected object tsf/patch-tekton-config-vbftd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:40.289721 3822 pods.go:176] Deleting pod: tsf/patch-tekton-config-vbftd I0326 23:45:40.290949 3822 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-vbftd, ips: 10.128.0.42 I0326 23:45:40.290974 3822 obj_retry.go:464] Detected object tsf/patch-tekton-config-vbftd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:41.163632 3822 cni.go:403] [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default] I0326 23:45:41.174989 3822 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:41.175017 3822 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:41.244111 3822 cni.go:424] [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-wddsf-zzkhg f671fd63f9e9a2ab2a4569886fb276b11e5c1c4414c59462c654503f5040561c network default NAD default], result "{}", err I0326 23:45:41.296313 3822 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:41.296347 3822 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-wddsf-zzkhg I0326 23:45:41.297651 3822 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-wddsf-zzkhg, ips: 10.128.0.40 I0326 23:45:41.297677 3822 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:42.189315 3822 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:42.189337 3822 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:45.278098 3822 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-864859dc95-jzjcc] creating logical port tsf-tas_rekor-redis-864859dc95-jzjcc for pod on switch ip-10-0-0-231.ec2.internal I0326 23:45:45.278202 3822 kube.go:256] Updating pod tsf-tas/rekor-redis-864859dc95-jzjcc I0326 23:45:45.294176 3822 pod.go:62] [tsf-tas/rekor-redis-864859dc95-jzjcc] pod update took 16.007189ms I0326 23:45:45.294198 3822 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-864859dc95-jzjcc] addLogicalPort annotation time took 16.035344ms I0326 23:45:45.295404 3822 pods.go:271] [tsf-tas/rekor-redis-864859dc95-jzjcc] addLogicalPort took 17.320047ms, libovsdb time 834.663µs I0326 23:45:46.234174 3822 cni.go:403] [tsf-tas/rekor-redis-864859dc95-jzjcc 115771c87f0a9c084e6ed7559b4f0fa47150e391d03153a6f355ba3b79514eb5 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-864859dc95-jzjcc 115771c87f0a9c084e6ed7559b4f0fa47150e391d03153a6f355ba3b79514eb5 network default NAD default] I0326 23:45:46.242644 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-864859dc95-jzjcc, hostIfaceName: 115771c87f0a9c0, network: default, NAD default, SandboxID: "115771c87f0a9c084e6ed7559b4f0fa47150e391d03153a6f355ba3b79514eb5", PCI device ID: , UID: "abc55bbe-3e94-4e01-becd-b025857b4e6a", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0326 23:45:46.494114 3822 cni.go:424] [tsf-tas/rekor-redis-864859dc95-jzjcc 115771c87f0a9c084e6ed7559b4f0fa47150e391d03153a6f355ba3b79514eb5 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-864859dc95-jzjcc 115771c87f0a9c084e6ed7559b4f0fa47150e391d03153a6f355ba3b79514eb5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:8e:ff:2e:ef:2c\",\"name\":\"115771c87f0a9c0\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58b97144-22a8-4cb4-870a-352b07ec9912\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:45:47.348618 3822 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-4gd4n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:47.349182 3822 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-4gd4n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:45:48.197954 3822 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:45:48.370720 3822 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-4gd4n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.370745 3822 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-4gd4n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.425763 3822 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.425787 3822 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.430726 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-vbftd I0326 23:45:48.430755 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-vbftd W0326 23:45:51.267049 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:45:52.845187 3822 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:52.845792 3822 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:45:53.268707 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:45:53.274469 3822 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:53.274490 3822 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:45:56.646809 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0326 23:45:58.447775 3822 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:58.448341 3822 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:58.466659 3822 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.466678 3822 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.474059 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0326 23:45:58.474083 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines W0326 23:46:00.238457 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:46:02.466443 3822 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:02.466976 3822 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:02.488524 3822 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:02.488543 3822 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:02.496163 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0326 23:46:02.496190 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0326 23:46:08.495387 3822 obj_retry.go:464] Detected object tsf-tpa/create-db-lqmnx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:08.495962 3822 obj_retry.go:464] Detected object tsf-tpa/create-db-lqmnx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:09.519795 3822 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:09.519819 3822 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:12.537162 3822 obj_retry.go:464] Detected object tsf-tpa/migrate-db-5nwz9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:12.537793 3822 obj_retry.go:464] Detected object tsf-tpa/migrate-db-5nwz9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:12.655539 3822 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:12.655558 3822 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:15.563851 3822 obj_retry.go:464] Detected object tsf-tpa/create-importers-stcj7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:15.564461 3822 obj_retry.go:464] Detected object tsf-tpa/create-importers-stcj7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:15.760088 3822 base_network_controller_pods.go:487] [default/tsf-tpa/server-84bf44886b-5gwwh] creating logical port tsf-tpa_server-84bf44886b-5gwwh for pod on switch ip-10-0-0-231.ec2.internal I0326 23:46:15.760205 3822 kube.go:256] Updating pod tsf-tpa/server-84bf44886b-5gwwh I0326 23:46:15.774376 3822 pod.go:62] [tsf-tpa/server-84bf44886b-5gwwh] pod update took 14.206483ms I0326 23:46:15.774405 3822 base_network_controller_pods.go:937] [default/tsf-tpa/server-84bf44886b-5gwwh] addLogicalPort annotation time took 14.239219ms I0326 23:46:15.775738 3822 pods.go:271] [tsf-tpa/server-84bf44886b-5gwwh] addLogicalPort took 15.671805ms, libovsdb time 870.042µs I0326 23:46:15.822590 3822 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-6mh9g] creating logical port tsf-tpa_importer-74cd48f5c7-6mh9g for pod on switch ip-10-0-0-231.ec2.internal I0326 23:46:15.822707 3822 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-6mh9g I0326 23:46:15.839239 3822 pod.go:62] [tsf-tpa/importer-74cd48f5c7-6mh9g] pod update took 16.550874ms I0326 23:46:15.839287 3822 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-6mh9g] addLogicalPort annotation time took 16.602077ms I0326 23:46:15.840686 3822 pods.go:271] [tsf-tpa/importer-74cd48f5c7-6mh9g] addLogicalPort took 18.118191ms, libovsdb time 918.461µs I0326 23:46:15.864357 3822 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:15.864380 3822 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:25.154410 3822 cni.go:403] [tsf-tpa/server-84bf44886b-5gwwh 420b0ff7b9d37b402e785c5944216423071aeb6a7a2e745b262db0b8691938b8 network default NAD default] ADD starting CNI request [tsf-tpa/server-84bf44886b-5gwwh 420b0ff7b9d37b402e785c5944216423071aeb6a7a2e745b262db0b8691938b8 network default NAD default] I0326 23:46:25.162502 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-84bf44886b-5gwwh, hostIfaceName: 420b0ff7b9d37b4, network: default, NAD default, SandboxID: "420b0ff7b9d37b402e785c5944216423071aeb6a7a2e745b262db0b8691938b8", PCI device ID: , UID: "fc89266d-6e5d-4511-a7a1-bea9ca006678", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0326 23:46:25.189409 3822 cni.go:403] [tsf-tpa/importer-74cd48f5c7-6mh9g b31933c0f568773877a8a5a426f56dbcf08ed0326ff611f11bb880ae856920fa network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-6mh9g b31933c0f568773877a8a5a426f56dbcf08ed0326ff611f11bb880ae856920fa network default NAD default] I0326 23:46:25.197684 3822 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-6mh9g, hostIfaceName: b31933c0f568773, network: default, NAD default, SandboxID: "b31933c0f568773877a8a5a426f56dbcf08ed0326ff611f11bb880ae856920fa", PCI device ID: , UID: "d78bbf09-8839-403a-b4c6-2694a62e2785", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0326 23:46:25.417141 3822 cni.go:424] [tsf-tpa/server-84bf44886b-5gwwh 420b0ff7b9d37b402e785c5944216423071aeb6a7a2e745b262db0b8691938b8 network default NAD default] ADD finished CNI request [tsf-tpa/server-84bf44886b-5gwwh 420b0ff7b9d37b402e785c5944216423071aeb6a7a2e745b262db0b8691938b8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:02:9c:a6:da:a9\",\"name\":\"420b0ff7b9d37b4\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a1817b72-447f-4860-9432-81fb68892a12\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:46:25.447948 3822 cni.go:424] [tsf-tpa/importer-74cd48f5c7-6mh9g b31933c0f568773877a8a5a426f56dbcf08ed0326ff611f11bb880ae856920fa network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-6mh9g b31933c0f568773877a8a5a426f56dbcf08ed0326ff611f11bb880ae856920fa network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:83:4e:bc:80:37\",\"name\":\"b31933c0f568773\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ed036c8-8c8a-41e8-a6f6-b55b214fd612\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:46:28.411401 3822 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0326 23:46:29.444129 3822 util.go:959] Failed to build node endpoints for node ip-10-0-0-231.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0326 23:46:31.633642 3822 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:31.634249 3822 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:31.655845 3822 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:31.655866 3822 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:31.664062 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0326 23:46:31.664088 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0326 23:46:32.419632 3822 namespace.go:98] [integration-service] adding namespace I0326 23:46:32.420587 3822 namespace.go:102] [integration-service] adding namespace took 933.695µs I0326 23:46:32.423239 3822 namespace.go:98] [build-service] adding namespace I0326 23:46:32.424018 3822 namespace.go:102] [build-service] adding namespace took 759.791µs I0326 23:46:32.430758 3822 namespace.go:98] [image-controller] adding namespace I0326 23:46:32.431386 3822 namespace.go:102] [image-controller] adding namespace took 611.905µs I0326 23:46:32.495284 3822 namespace.go:142] [integration-service] updating namespace I0326 23:46:32.537385 3822 namespace.go:142] [integration-service] updating namespace I0326 23:46:32.567117 3822 namespace.go:142] [build-service] updating namespace I0326 23:46:32.594225 3822 namespace.go:142] [build-service] updating namespace I0326 23:46:32.610284 3822 namespace.go:142] [build-service] updating namespace I0326 23:46:32.626521 3822 namespace.go:142] [image-controller] updating namespace I0326 23:46:32.626605 3822 namespace.go:142] [integration-service] updating namespace I0326 23:46:32.645349 3822 namespace.go:142] [image-controller] updating namespace I0326 23:46:32.662126 3822 namespace.go:98] [release-service] adding namespace I0326 23:46:32.662940 3822 namespace.go:102] [release-service] adding namespace took 799.599µs I0326 23:46:32.673664 3822 namespace.go:142] [konflux-ui] updating namespace I0326 23:46:32.691136 3822 namespace.go:142] [release-service] updating namespace I0326 23:46:32.706901 3822 namespace.go:142] [release-service] updating namespace I0326 23:46:32.797800 3822 namespace.go:98] [konflux-info] adding namespace I0326 23:46:32.798481 3822 namespace.go:102] [konflux-info] adding namespace took 668.213µs I0326 23:46:32.872019 3822 namespace.go:142] [konflux-info] updating namespace I0326 23:46:32.872040 3822 namespace.go:98] [namespace-lister] adding namespace I0326 23:46:32.872789 3822 namespace.go:102] [namespace-lister] adding namespace took 734.578µs I0326 23:46:32.889227 3822 namespace.go:98] [enterprise-contract-service] adding namespace I0326 23:46:32.890068 3822 namespace.go:102] [enterprise-contract-service] adding namespace took 822.949µs I0326 23:46:32.919291 3822 namespace.go:142] [image-controller] updating namespace I0326 23:46:33.104303 3822 namespace.go:142] [konflux-info] updating namespace I0326 23:46:33.104411 3822 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:33.195436 3822 namespace.go:98] [default-tenant] adding namespace I0326 23:46:33.196245 3822 namespace.go:102] [default-tenant] adding namespace took 788.431µs I0326 23:46:33.244468 3822 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:33.300431 3822 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:33.370445 3822 namespace.go:98] [segment-bridge] adding namespace I0326 23:46:33.370571 3822 namespace.go:142] [default-tenant] updating namespace I0326 23:46:33.371275 3822 namespace.go:102] [segment-bridge] adding namespace took 794.058µs I0326 23:46:33.480319 3822 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:33.642396 3822 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:33.642474 3822 namespace.go:142] [default-tenant] updating namespace I0326 23:46:33.764116 3822 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:34.153089 3822 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0326 23:46:34.153689 3822 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0326 23:46:34.153707 3822 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0326 23:46:34.156536 3822 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0326 23:46:34.208372 3822 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0326 23:46:34.208398 3822 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0326 23:46:34.209170 3822 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0326 23:46:34.699157 3822 namespace.go:142] [build-service] updating namespace I0326 23:46:35.096685 3822 namespace.go:142] [default-tenant] updating namespace I0326 23:46:35.407875 3822 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:35.673349 3822 namespace.go:142] [image-controller] updating namespace I0326 23:46:35.894034 3822 namespace.go:142] [integration-service] updating namespace I0326 23:46:36.149889 3822 namespace.go:142] [konflux-info] updating namespace I0326 23:46:36.383217 3822 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:36.579164 3822 namespace.go:142] [release-service] updating namespace I0326 23:46:36.777511 3822 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:36.879677 3822 namespace.go:142] [build-service] updating namespace I0326 23:46:36.899188 3822 namespace.go:142] [default-tenant] updating namespace I0326 23:46:36.917286 3822 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:36.929952 3822 namespace.go:142] [image-controller] updating namespace I0326 23:46:36.950825 3822 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:36.951521 3822 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:36.954211 3822 namespace.go:142] [integration-service] updating namespace I0326 23:46:36.977744 3822 namespace.go:142] [konflux-info] updating namespace I0326 23:46:36.998012 3822 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:37.008061 3822 namespace.go:142] [release-service] updating namespace I0326 23:46:37.032272 3822 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:37.076200 3822 namespace.go:142] [build-service] updating namespace I0326 23:46:37.121365 3822 namespace.go:142] [default-tenant] updating namespace I0326 23:46:37.206174 3822 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:37.306560 3822 namespace.go:142] [image-controller] updating namespace I0326 23:46:37.405378 3822 namespace.go:142] [integration-service] updating namespace I0326 23:46:37.506150 3822 namespace.go:142] [konflux-info] updating namespace W0326 23:46:37.554311 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0326 23:46:37.605679 3822 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:37.652881 3822 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:37.652900 3822 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:37.704819 3822 namespace.go:142] [release-service] updating namespace I0326 23:46:37.805599 3822 namespace.go:142] [segment-bridge] updating namespace W0326 23:46:37.888940 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0326 23:46:39.897945 3822 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0326 23:46:41.923139 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:46:48.823227 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=78 W0326 23:46:49.963933 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:46:49.966053 3822 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:46:49.991701 3822 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:46:49.991720 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy I0326 23:46:50.958096 3822 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:50.958629 3822 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:51.982315 3822 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:51.982337 3822 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:46:52.674228 3822 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:46:52.675305 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:46:52.700648 3822 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:46:52.700662 3822 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:46:52.728538 3822 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0326 23:46:52.754407 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:46:52.755582 3822 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0326 23:47:01.290233 3822 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:01.290823 3822 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:01.737596 3822 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:01.737619 3822 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:01.743529 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-h2lcc I0326 23:47:01.743558 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-h2lcc W0326 23:47:02.748872 3822 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0326 23:47:02.827728 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=156 I0326 23:47:32.929637 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0326 23:47:46.832828 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=691 I0326 23:47:58.352768 3822 obj_retry.go:464] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:58.353378 3822 obj_retry.go:464] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:58.682631 3822 obj_retry.go:458] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:58.682659 3822 obj_retry.go:458] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:58.689715 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-7b5fc4779b-vk52h I0326 23:47:58.689742 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-7b5fc4779b-vk52h I0326 23:48:08.286050 3822 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:08.286710 3822 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:08.326178 3822 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:08.326197 3822 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:08.337607 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0326 23:48:08.337630 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0326 23:48:28.824110 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0326 23:48:33.681441 3822 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default] I0326 23:48:33.760736 3822 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 d41c78de478a8c324a62ee17c9fa9db2e72565de9e32843bfabd87c671423119 network default NAD default], result "{}", err I0326 23:48:33.815839 3822 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:33.815877 3822 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 I0326 23:48:33.817242 3822 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2, ips: 10.128.0.27 I0326 23:48:33.817287 3822 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:34.022184 3822 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:34.022211 3822 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:34.027919 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 I0326 23:48:34.027952 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 I0326 23:48:38.985044 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0326 23:48:46.952210 3822 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:46.952811 3822 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:47.676796 3822 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:47.676818 3822 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:47.683993 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg I0326 23:48:47.684020 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg I0326 23:48:53.949520 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0326 23:49:59.829031 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=32 I0326 23:50:04.293460 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.294019 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:05.159985 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.160013 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:14.918147 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0326 23:50:52.915080 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0326 23:51:40.833461 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=77 I0326 23:52:32.947740 3822 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0326 23:52:53.953972 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 W0326 23:53:29.486141 3822 duration.go:461] Config duration recorder: measurement expired for pod/openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:53:33.917110 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0326 23:53:50.942015 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0326 23:54:21.831983 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0326 23:54:33.927232 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:54:48.953515 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0326 23:55:38.836492 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=43 I0326 23:55:56.832575 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=31 I0326 23:55:57.826779 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=12 I0326 23:56:06.827810 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0326 23:56:09.049225 3822 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:09.049288 3822 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:09.054620 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-lqmnx I0326 23:56:09.054651 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-lqmnx I0326 23:56:12.031406 3822 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:12.031460 3822 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:12.034962 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-5nwz9 I0326 23:56:12.034989 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-5nwz9 I0326 23:56:15.032612 3822 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:15.032655 3822 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:15.036996 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-stcj7 I0326 23:56:15.037025 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-stcj7 I0326 23:56:26.988495 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:56:58.932846 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0326 23:57:08.585394 3822 namespace.go:98] [default-tenant-managed] adding namespace I0326 23:57:08.586342 3822 namespace.go:102] [default-tenant-managed] adding namespace took 923.823µs I0326 23:57:08.610618 3822 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:08.628007 3822 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:09.669674 3822 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:09.802518 3822 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:09.901675 3822 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:13.922027 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0326 23:57:55.918495 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0326 23:58:12.262274 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:12.262902 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.805912 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.806509 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:54.921035 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0326 23:58:57.959681 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:57.960224 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:59:10.957864 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0327 00:00:02.964184 3822 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:02.964810 3822 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.849983 3822 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.850467 3822 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.976907 3822 obj_retry.go:464] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.977512 3822 obj_retry.go:464] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.988869 3822 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:03.988891 3822 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.870605 3822 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.870628 3822 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.982634 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.983255 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.997025 3822 obj_retry.go:458] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.997050 3822 obj_retry.go:458] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.003472 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.003496 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.031490 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.031520 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.035911 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 I0327 00:00:06.035937 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 I0327 00:00:14.968665 3822 obj_retry.go:464] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:14.969333 3822 obj_retry.go:464] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:15.991673 3822 obj_retry.go:458] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:15.991721 3822 obj_retry.go:458] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:19.957546 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 00:00:27.945666 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0327 00:00:29.951463 3822 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0327 00:00:42.836711 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=16 I0327 00:00:54.337506 3822 obj_retry.go:464] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:54.338058 3822 obj_retry.go:464] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:55.195975 3822 obj_retry.go:458] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:55.195998 3822 obj_retry.go:458] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:01:32.930676 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 00:01:38.014726 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:01:38.015344 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:06.936696 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0327 00:02:21.071131 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:21.071692 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:23.524913 3822 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod for pod on switch ip-10-0-0-231.ec2.internal I0327 00:02:23.525039 3822 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod I0327 00:02:23.549554 3822 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod] pod update took 24.548577ms I0327 00:02:23.549579 3822 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod] addLogicalPort annotation time took 24.578476ms I0327 00:02:23.550826 3822 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod] addLogicalPort took 25.934048ms, libovsdb time 838.42µs I0327 00:02:23.835704 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=46 I0327 00:02:23.882929 3822 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default] I0327 00:02:23.891437 3822 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod, hostIfaceName: 223e8c683fd4029, network: default, NAD default, SandboxID: "223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788", PCI device ID: , UID: "903d4691-dcaf-4a4f-9839-a05882a0ccee", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0327 00:02:24.145400 3822 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:80:08:ef:65:9a\",\"name\":\"223e8c683fd4029\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/357afefa-d874-4e70-86c6-f2ace1121d37\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 00:02:34.835650 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 I0327 00:03:04.712317 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:04.712912 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:05.462106 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:05.462711 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:19.675316 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:19.675853 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:24.798286 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:24.798837 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:24.839925 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=279 I0327 00:03:28.572210 3822 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default] I0327 00:03:28.669634 3822 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod 223e8c683fd40298965593524f728a0c1890a4035c707bfe19d2fdcaeec52788 network default NAD default], result "{}", err I0327 00:03:28.744956 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.744989 3822 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod I0327 00:03:28.746417 3822 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod, ips: 10.128.0.46 I0327 00:03:28.746448 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.822752 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.823286 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.861356 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.861904 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.894618 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.895136 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.921870 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0327 00:03:37.830812 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0327 00:03:53.014095 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:53.014671 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:55.831086 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0327 00:04:12.116254 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:12.116870 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:27.962028 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0327 00:04:53.991809 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 00:05:39.496579 3822 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:39.497093 3822 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:39.646028 3822 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:39.646052 3822 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:39.649451 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-multus/cni-sysctl-allowlist-ds-jnwzv I0327 00:05:39.649480 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-multus/cni-sysctl-allowlist-ds-jnwzv I0327 00:05:40.170813 3822 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.170860 3822 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.255897 3822 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.255929 3822 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.261441 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-multus/cni-sysctl-allowlist-ds-zs4tx I0327 00:05:40.261468 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-multus/cni-sysctl-allowlist-ds-zs4tx I0327 00:05:40.420735 3822 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.421221 3822 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.544576 3822 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.544608 3822 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.551399 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-multus/cni-sysctl-allowlist-ds-w5r9x I0327 00:05:40.551423 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-multus/cni-sysctl-allowlist-ds-w5r9x I0327 00:05:46.767309 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:46.767966 3822 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:56.949009 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0327 00:06:21.961445 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0327 00:06:38.158441 3822 obj_retry.go:464] Detected object default-tenant/my-integration-test-xpec-klhcb-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:38.159018 3822 obj_retry.go:464] Detected object default-tenant/my-integration-test-xpec-klhcb-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:50.925395 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0327 00:07:23.940621 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0327 00:07:30.598190 3822 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 00:08:32.925246 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0327 00:08:54.955120 3822 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 00:08:57.839883 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=49 I0327 00:09:06.839804 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0327 00:09:07.925825 3822 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0327 00:09:49.841160 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=10 I0327 00:09:52.833999 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0327 00:10:04.868827 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:04.869454 3822 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:05.707232 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.707271 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.738201 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.738248 3822 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.743718 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 I0327 00:10:05.743747 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 I0327 00:10:25.934455 3822 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0327 00:10:37.834563 3822 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0327 00:11:03.249189 3822 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.249813 3822 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.311748 3822 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.312407 3822 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.368997 3822 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.369671 3822 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.778886 3822 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:03.778938 3822 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:03.787366 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj I0327 00:11:03.787399 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj I0327 00:11:04.169407 3822 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.169449 3822 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.176444 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq I0327 00:11:04.176472 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq I0327 00:11:04.203292 3822 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.203347 3822 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.207678 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j I0327 00:11:04.207703 3822 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j