+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-188.ec2.internal ++ [[ -n ip-10-0-2-188.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-188.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0331 03:54:42.906252192 - disable conntrack on geneve port' I0331 03:54:42.906252192 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0331 03:54:42.923346978 - starting ovnkube-node + echo 'I0331 03:54:42.923346978 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-188.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-188.ec2.internal --init-node ip-10-0-2-188.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-188.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0331 03:54:42.963297 3904 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0331 03:54:42.963342 3904 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-f465bfb369.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0331 03:54:42.964757 3904 kube.go:419] Waiting for certificate I0331 03:54:42.964801 3904 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0331 03:54:42.964850 3904 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0331 03:54:42.965625 3904 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:54:43.014754 3904 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:54:43.014771 3904 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:54:43.020865 3904 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:54:43.020947 3904 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5dw95" I0331 03:54:43.026619 3904 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5dw95" I0331 03:54:43.026688 3904 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:54:43.965584 3904 kube.go:426] Certificate found I0331 03:54:43.966236 3904 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:54:43.967026 3904 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0331 03:54:44.029633 3904 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-01 03:49:43 +0000 UTC" deadline="2026-03-31 22:52:45.330779226 +0000 UTC" I0331 03:54:44.029670 3904 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h58m1.301122137s" I0331 03:54:44.033038 3904 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:54:44.043346 3904 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0331 03:54:44.043436 3904 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0331 03:54:44.043587 3904 factory.go:531] Starting watch factory I0331 03:54:44.043674 3904 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.043694 3904 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.043956 3904 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.043970 3904 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.044144 3904 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.044159 3904 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.044310 3904 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.044332 3904 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.044527 3904 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.044595 3904 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.045004 3904 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.045042 3904 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.052430 3904 controller_manager.go:368] Starting the ovnkube controller I0331 03:54:44.052510 3904 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-188.ec2.internal I0331 03:54:44.052575 3904 controller_manager.go:393] NBDB zone sync took: 50.52µs I0331 03:54:44.052624 3904 factory.go:531] Starting watch factory I0331 03:54:44.068560 3904 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0331 03:54:44.068670 3904 ovn_northd.go:121] Registering OVN northd metrics I0331 03:54:44.068720 3904 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.068757 3904 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.068777 3904 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.068812 3904 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.069475 3904 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.072618 3904 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0331 03:54:44.072672 3904 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0331 03:54:44.077921 3904 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:54:44.088422 3904 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0331 03:54:44.149555 3904 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:54:44.149570 3904 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:54:44.149574 3904 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:54:44.149577 3904 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:54:44.149580 3904 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:54:44.149583 3904 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:54:44.149651 3904 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:54:44.149668 3904 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:54:44.149674 3904 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:54:44.149688 3904 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:54:44.152652 3904 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:54:44.152654 3904 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:54:44.155063 3904 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:54:44.155074 3904 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:54:44.155133 3904 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.155150 3904 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.157672 3904 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.157715 3904 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:54:44.157726 3904 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:54:44.157729 3904 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:54:44.157732 3904 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:54:44.157735 3904 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:54:44.157737 3904 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:54:44.161028 3904 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:54:44.161079 3904 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.161096 3904 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.163572 3904 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:54:44.163584 3904 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:54:44.163704 3904 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.167100 3904 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:54:44.167231 3904 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.167272 3904 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.169219 3904 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:54:44.169858 3904 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.172174 3904 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:54:44.172290 3904 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.172330 3904 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.174304 3904 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:54:44.175004 3904 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.177479 3904 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:54:44.177532 3904 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.177545 3904 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.179710 3904 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:54:44.180183 3904 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.183121 3904 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:54:44.183289 3904 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:54:44.183300 3904 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:54:44.185398 3904 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:54:44.187694 3904 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:54:44.188631 3904 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:54:44.188698 3904 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.188709 3904 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.188748 3904 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.188764 3904 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.191116 3904 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:54:44.192981 3904 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.193024 3904 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:54:44.193907 3904 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:54:44.193916 3904 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:54:44.193922 3904 factory.go:636] Watch Factory start up complete, took: 150.338763ms I0331 03:54:44.193980 3904 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0331 03:54:44.197073 3904 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:54:44.202696 3904 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:54:44.202705 3904 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:54:44.202712 3904 factory.go:636] Watch Factory start up complete, took: 150.088814ms I0331 03:54:44.202720 3904 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-188.ec2.internal" zone I0331 03:54:44.238388 3904 default_node_network_controller.go:709] Initializing the default node network controller I0331 03:54:44.292749 3904 udn_isolation.go:101] Starting UDN host isolation manager I0331 03:54:44.299666 3904 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0331 03:54:44.364381 3904 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0331 03:54:44.364472 3904 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0331 03:54:44.364490 3904 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0331 03:54:44.397600 3904 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0331 03:54:44.397662 3904 default_node_network_controller.go:808] Node ip-10-0-2-188.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0331 03:54:44.403438 3904 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.188"] k8s.ovn.org/zone-name:ip-10-0-2-188.ec2.internal] on node ip-10-0-2-188.ec2.internal I0331 03:54:44.423497 3904 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0331 03:54:44.428998 3904 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0331 03:54:44.429137 3904 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0331 03:54:44.429306 3904 gateway_init.go:263] Preparing Gateway I0331 03:54:44.429319 3904 gateway_shared_intf.go:1658] Creating new gateway I0331 03:54:44.452930 3904 controller_manager.go:427] Waiting for node in zone sync took: 250.197675ms I0331 03:54:44.461021 3904 controller_manager.go:608] Switching to transit router for layer2 networks I0331 03:54:44.461036 3904 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-188.ec2.internal I0331 03:54:44.476503 3904 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:54:44.482298 3904 controller_manager.go:288] SCTP support detected in OVN I0331 03:54:44.482985 3904 services_controller.go:69] Creating services controller for network=default I0331 03:54:44.483123 3904 udn_enabled_svc.go:80] Waiting for service informer to sync I0331 03:54:44.483395 3904 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0331 03:54:44.483436 3904 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0331 03:54:44.483456 3904 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0331 03:54:44.483470 3904 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:54:44.483683 3904 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0331 03:54:44.483727 3904 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0331 03:54:44.483756 3904 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 20.777µs I0331 03:54:44.483785 3904 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 37.632µs I0331 03:54:44.483790 3904 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 69.493µs I0331 03:54:44.483796 3904 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0331 03:54:44.483804 3904 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0331 03:54:44.483797 3904 udn_enabled_svc.go:95] Performing full resync I0331 03:54:44.483813 3904 default_network_controller.go:321] Starting the default network controller I0331 03:54:44.483894 3904 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 17.695µs I0331 03:54:44.484144 3904 udn_enabled_svc.go:99] Waiting for handler to sync I0331 03:54:44.484158 3904 udn_enabled_svc.go:104] Starting worker I0331 03:54:44.493562 3904 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0331 03:54:44.493670 3904 repair.go:33] Syncing exgw routes took 88.24µs I0331 03:54:44.493703 3904 default_network_controller.go:407] Starting all the Watchers... I0331 03:54:44.493947 3904 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0331 03:54:44.493960 3904 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0331 03:54:44.493962 3904 namespace.go:98] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] adding namespace I0331 03:54:44.493962 3904 namespace.go:98] [openshift-config-managed] adding namespace I0331 03:54:44.493947 3904 namespace.go:98] [openshift-multus] adding namespace I0331 03:54:44.493975 3904 namespace.go:98] [openshift-backplane-cee] adding namespace I0331 03:54:44.493983 3904 namespace.go:98] [openshift-dns-operator] adding namespace I0331 03:54:44.493986 3904 namespace.go:98] [openshift-customer-monitoring] adding namespace I0331 03:54:44.493987 3904 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0331 03:54:44.493986 3904 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0331 03:54:44.494000 3904 namespace.go:98] [openshift-operators] adding namespace I0331 03:54:44.494004 3904 namespace.go:98] [openshift-cluster-version] adding namespace I0331 03:54:44.494004 3904 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0331 03:54:44.494019 3904 namespace.go:98] [kube-system] adding namespace I0331 03:54:44.493947 3904 namespace.go:98] [openshift-network-operator] adding namespace I0331 03:54:44.494741 3904 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 777.971µs I0331 03:54:44.494763 3904 namespace.go:98] [openshift-apiserver-operator] adding namespace I0331 03:54:44.495311 3904 namespace.go:102] [openshift-apiserver-operator] adding namespace took 540.224µs I0331 03:54:44.495328 3904 namespace.go:98] [kube-node-lease] adding namespace I0331 03:54:44.495872 3904 namespace.go:102] [kube-node-lease] adding namespace took 535.087µs I0331 03:54:44.495894 3904 namespace.go:98] [openshift-authentication] adding namespace I0331 03:54:44.496442 3904 namespace.go:102] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] adding namespace took 2.469906ms I0331 03:54:44.496457 3904 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0331 03:54:44.496923 3904 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 2.953289ms I0331 03:54:44.496938 3904 namespace.go:98] [openshift-image-registry] adding namespace W0331 03:54:44.497412 3904 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-lk8gr: pod openshift-multus/network-metrics-daemon-lk8gr: no pod IPs found I0331 03:54:44.497441 3904 namespace.go:102] [openshift-config-managed] adding namespace took 3.466215ms I0331 03:54:44.497451 3904 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0331 03:54:44.498091 3904 namespace.go:102] [openshift-multus] adding namespace took 4.113652ms I0331 03:54:44.498112 3904 namespace.go:98] [openshift-service-ca-operator] adding namespace I0331 03:54:44.498633 3904 namespace.go:102] [openshift-backplane-cee] adding namespace took 4.64479ms I0331 03:54:44.498651 3904 namespace.go:98] [kube-public] adding namespace I0331 03:54:44.499114 3904 namespace.go:102] [openshift-dns-operator] adding namespace took 5.122785ms I0331 03:54:44.499125 3904 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0331 03:54:44.499594 3904 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.600619ms I0331 03:54:44.499608 3904 namespace.go:98] [openshift-ingress] adding namespace I0331 03:54:44.500057 3904 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.061084ms I0331 03:54:44.500078 3904 namespace.go:98] [openshift] adding namespace I0331 03:54:44.500604 3904 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.602232ms I0331 03:54:44.500619 3904 namespace.go:98] [openshift-network-node-identity] adding namespace I0331 03:54:44.501098 3904 namespace.go:102] [openshift-operators] adding namespace took 7.092746ms I0331 03:54:44.501107 3904 namespace.go:98] [openshift-machine-config-operator] adding namespace I0331 03:54:44.501612 3904 namespace.go:102] [openshift-cluster-version] adding namespace took 7.598488ms I0331 03:54:44.501625 3904 namespace.go:98] [openshift-package-operator] adding namespace I0331 03:54:44.502057 3904 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.03978ms I0331 03:54:44.502074 3904 namespace.go:98] [openshift-kube-scheduler] adding namespace I0331 03:54:44.502545 3904 namespace.go:102] [kube-system] adding namespace took 8.516568ms I0331 03:54:44.502563 3904 namespace.go:98] [openshift-apiserver] adding namespace I0331 03:54:44.503092 3904 namespace.go:102] [openshift-network-operator] adding namespace took 9.02351ms I0331 03:54:44.503106 3904 namespace.go:98] [openshift-console-operator] adding namespace I0331 03:54:44.503609 3904 namespace.go:102] [openshift-authentication] adding namespace took 7.709255ms I0331 03:54:44.503622 3904 namespace.go:98] [openshift-console] adding namespace I0331 03:54:44.504046 3904 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.583404ms I0331 03:54:44.504059 3904 namespace.go:98] [openshift-dns] adding namespace I0331 03:54:44.504502 3904 namespace.go:102] [openshift-image-registry] adding namespace took 7.559486ms I0331 03:54:44.504513 3904 namespace.go:98] [openshift-kube-apiserver] adding namespace I0331 03:54:44.504922 3904 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.466777ms I0331 03:54:44.504932 3904 namespace.go:98] [openshift-node] adding namespace I0331 03:54:44.505337 3904 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.218417ms I0331 03:54:44.505352 3904 namespace.go:98] [openshift-ingress-operator] adding namespace I0331 03:54:44.505745 3904 namespace.go:102] [kube-public] adding namespace took 7.089175ms I0331 03:54:44.505757 3904 namespace.go:98] [openshift-operators-redhat] adding namespace I0331 03:54:44.506174 3904 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.024481ms I0331 03:54:44.506187 3904 namespace.go:98] [openshift-logging] adding namespace I0331 03:54:44.506553 3904 namespace.go:102] [openshift-ingress] adding namespace took 6.940223ms I0331 03:54:44.506566 3904 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0331 03:54:44.506916 3904 namespace.go:102] [openshift] adding namespace took 6.83317ms I0331 03:54:44.506926 3904 namespace.go:98] [openshift-backplane-tam] adding namespace I0331 03:54:44.507417 3904 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.792905ms I0331 03:54:44.507429 3904 namespace.go:98] [openshift-backplane-csm] adding namespace I0331 03:54:44.507778 3904 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.664621ms I0331 03:54:44.507795 3904 namespace.go:98] [openshift-console-user-settings] adding namespace I0331 03:54:44.508219 3904 namespace.go:102] [openshift-package-operator] adding namespace took 6.587296ms I0331 03:54:44.508234 3904 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0331 03:54:44.508680 3904 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.600611ms I0331 03:54:44.508692 3904 namespace.go:98] [openshift-route-controller-manager] adding namespace I0331 03:54:44.509076 3904 namespace.go:102] [openshift-apiserver] adding namespace took 6.507351ms I0331 03:54:44.509088 3904 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0331 03:54:44.509535 3904 namespace.go:102] [openshift-console-operator] adding namespace took 6.424671ms I0331 03:54:44.509548 3904 namespace.go:98] [openshift-backplane] adding namespace I0331 03:54:44.509978 3904 namespace.go:102] [openshift-console] adding namespace took 6.348928ms I0331 03:54:44.509998 3904 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0331 03:54:44.510452 3904 namespace.go:102] [openshift-dns] adding namespace took 6.388051ms I0331 03:54:44.510468 3904 namespace.go:98] [openshift-authentication-operator] adding namespace I0331 03:54:44.510695 3904 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:54:44.511007 3904 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.488129ms I0331 03:54:44.511020 3904 namespace.go:98] [openshift-etcd] adding namespace I0331 03:54:44.511469 3904 namespace.go:102] [openshift-node] adding namespace took 6.532428ms I0331 03:54:44.511480 3904 namespace.go:98] [openshift-backplane-srep] adding namespace I0331 03:54:44.511855 3904 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.495294ms I0331 03:54:44.511874 3904 namespace.go:98] [default] adding namespace I0331 03:54:44.512399 3904 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.636245ms I0331 03:54:44.512414 3904 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0331 03:54:44.512896 3904 namespace.go:102] [openshift-logging] adding namespace took 6.705808ms I0331 03:54:44.512903 3904 namespace.go:98] [openshift-insights] adding namespace I0331 03:54:44.513556 3904 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.982417ms I0331 03:54:44.513579 3904 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0331 03:54:44.514069 3904 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.138286ms I0331 03:54:44.514080 3904 namespace.go:98] [openshift-infra] adding namespace I0331 03:54:44.514568 3904 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.132866ms I0331 03:54:44.514582 3904 namespace.go:98] [openshift-network-diagnostics] adding namespace I0331 03:54:44.515108 3904 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.308446ms I0331 03:54:44.515119 3904 namespace.go:98] [openshift-backplane-mobb] adding namespace I0331 03:54:44.515633 3904 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.394483ms I0331 03:54:44.515647 3904 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0331 03:54:44.516160 3904 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.463132ms I0331 03:54:44.516172 3904 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0331 03:54:44.516712 3904 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.618687ms I0331 03:54:44.516726 3904 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0331 03:54:44.517134 3904 namespace.go:102] [openshift-backplane] adding namespace took 7.58102ms I0331 03:54:44.517148 3904 namespace.go:98] [openshift-marketplace] adding namespace I0331 03:54:44.517498 3904 iptables.go:108] Creating table: filter chain: OUTPUT I0331 03:54:44.517570 3904 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.567179ms I0331 03:54:44.518120 3904 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.634209ms I0331 03:54:44.518132 3904 namespace.go:98] [openshift-machine-api] adding namespace I0331 03:54:44.518573 3904 namespace.go:102] [openshift-etcd] adding namespace took 7.547985ms I0331 03:54:44.518586 3904 namespace.go:98] [openshift-network-console] adding namespace I0331 03:54:44.519017 3904 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.524095ms I0331 03:54:44.519029 3904 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0331 03:54:44.519602 3904 namespace.go:102] [default] adding namespace took 7.720622ms I0331 03:54:44.519616 3904 namespace.go:98] [openshift-config] adding namespace I0331 03:54:44.520038 3904 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.61991ms I0331 03:54:44.520048 3904 namespace.go:98] [openshift-ingress-canary] adding namespace I0331 03:54:44.520906 3904 namespace.go:102] [openshift-insights] adding namespace took 7.997719ms I0331 03:54:44.520918 3904 namespace.go:98] [openshift-monitoring] adding namespace I0331 03:54:44.521416 3904 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.830289ms I0331 03:54:44.521430 3904 namespace.go:98] [openshift-controller-manager] adding namespace W0331 03:54:44.521829 3904 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-fsh2n: pod openshift-network-diagnostics/network-check-target-fsh2n: no pod IPs found I0331 03:54:44.521866 3904 namespace.go:102] [openshift-infra] adding namespace took 7.781237ms I0331 03:54:44.521878 3904 namespace.go:98] [dedicated-admin] adding namespace I0331 03:54:44.522303 3904 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.716675ms I0331 03:54:44.522316 3904 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0331 03:54:44.522698 3904 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.573998ms I0331 03:54:44.523142 3904 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.489626ms I0331 03:54:44.523157 3904 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0331 03:54:44.523582 3904 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.405454ms I0331 03:54:44.523595 3904 namespace.go:98] [openshift-backplane-cse] adding namespace I0331 03:54:44.524016 3904 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.283302ms I0331 03:54:44.524030 3904 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0331 03:54:44.524327 3904 gateway_shared_intf.go:1787] Gateway Creation Complete I0331 03:54:44.524340 3904 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-188.ec2.internal","mac-address":"0e:d2:1c:d7:d3:bb","ip-addresses":["10.0.2.188/24"],"ip-address":"10.0.2.188/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:a2479525-91e9-424b-9527-fe29c4ce6529 k8s.ovn.org/node-encap-ips:["10.0.2.188"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.188/24"} k8s.ovn.org/zone-name:ip-10-0-2-188.ec2.internal] on node ip-10-0-2-188.ec2.internal I0331 03:54:44.524447 3904 namespace.go:102] [openshift-marketplace] adding namespace took 7.294704ms I0331 03:54:44.524458 3904 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0331 03:54:44.524870 3904 namespace.go:102] [openshift-machine-api] adding namespace took 6.73187ms I0331 03:54:44.525301 3904 namespace.go:102] [openshift-network-console] adding namespace took 6.709546ms I0331 03:54:44.525313 3904 namespace.go:98] [openshift-config-operator] adding namespace I0331 03:54:44.525672 3904 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.638879ms I0331 03:54:44.526030 3904 namespace.go:102] [openshift-config] adding namespace took 6.408703ms I0331 03:54:44.526043 3904 namespace.go:98] [openshift-host-network] adding namespace I0331 03:54:44.526427 3904 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.375154ms I0331 03:54:44.526853 3904 namespace.go:102] [openshift-monitoring] adding namespace took 5.929203ms I0331 03:54:44.527225 3904 namespace.go:102] [openshift-controller-manager] adding namespace took 5.788895ms I0331 03:54:44.527668 3904 namespace.go:102] [dedicated-admin] adding namespace took 5.785559ms I0331 03:54:44.527681 3904 namespace.go:98] [openshift-must-gather-operator] adding namespace I0331 03:54:44.528061 3904 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 5.740515ms I0331 03:54:44.528518 3904 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.354864ms I0331 03:54:44.528890 3904 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.289505ms I0331 03:54:44.528902 3904 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0331 03:54:44.529215 3904 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 5.179853ms I0331 03:54:44.529676 3904 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.213666ms I0331 03:54:44.530079 3904 namespace.go:102] [openshift-config-operator] adding namespace took 4.761892ms I0331 03:54:44.530538 3904 namespace.go:102] [openshift-host-network] adding namespace took 4.488922ms I0331 03:54:44.530950 3904 namespace.go:102] [openshift-must-gather-operator] adding namespace took 3.263552ms I0331 03:54:44.531343 3904 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 2.436581ms I0331 03:54:44.531682 3904 zone_ic_handler.go:186] Time taken to create transit switch: 251.33µs I0331 03:54:44.531752 3904 master.go:532] Adding or Updating local node "ip-10-0-2-188.ec2.internal" for network "default" I0331 03:54:44.534662 3904 default_network_controller.go:806] Node add failed for ip-10-0-2-188.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-188.ec2.internal, error getting gateway config for node ip-10-0-2-188.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-188.ec2.internal", failed to update chassis to local for local node ip-10-0-2-188.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-188.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-188.ec2.internal] E0331 03:54:44.534715 3904 obj_retry.go:541] Failed to create *v1.Node ip-10-0-2-188.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-188.ec2.internal, error getting gateway config for node ip-10-0-2-188.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-188.ec2.internal", failed to update chassis to local for local node ip-10-0-2-188.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-188.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-188.ec2.internal] I0331 03:54:44.534760 3904 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0331 03:54:44.534782 3904 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0331 03:54:44.534779 3904 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-188.ec2.internal", UID:"260b2c7b-c245-497c-9032-94c238dba3ba", APIVersion:"v1", ResourceVersion:"8064", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-188.ec2.internal, error getting gateway config for node ip-10-0-2-188.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-188.ec2.internal", failed to update chassis to local for local node ip-10-0-2-188.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-188.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-188.ec2.internal] I0331 03:54:44.534812 3904 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0331 03:54:44.534827 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-188.ec2.internal in network "default" I0331 03:54:44.534855 3904 node_tracker.go:235] Node ip-10-0-2-188.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-188.ec2.internal" I0331 03:54:44.534868 3904 node_tracker.go:169] Node ip-10-0-2-188.ec2.internal switch + router changed, syncing services in network "default" I0331 03:54:44.534878 3904 services_controller.go:574] Full service sync requested for network=default I0331 03:54:44.538995 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-188.ec2.internal in network "default" I0331 03:54:44.539040 3904 node_tracker.go:169] Node ip-10-0-2-188.ec2.internal switch + router changed, syncing services in network "default" I0331 03:54:44.539051 3904 services_controller.go:574] Full service sync requested for network=default I0331 03:54:44.539082 3904 master.go:532] Adding or Updating local node "ip-10-0-2-188.ec2.internal" for network "default" I0331 03:54:44.541926 3904 default_node_network_controller.go:518] Node connection status = connected W0331 03:54:44.543555 3904 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-lk8gr: pod openshift-multus/network-metrics-daemon-lk8gr: no pod IPs found W0331 03:54:44.543571 3904 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-fsh2n: pod openshift-network-diagnostics/network-check-target-fsh2n: no pod IPs found I0331 03:54:44.544474 3904 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-188.ec2.internal for the network default I0331 03:54:44.635388 3904 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0331 03:54:44.635411 3904 services_controller.go:208] Setting up event handlers for services for network=default I0331 03:54:44.635487 3904 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0331 03:54:44.635520 3904 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0331 03:54:44.635531 3904 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0331 03:54:44.736189 3904 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0331 03:54:44.736228 3904 repair.go:60] Starting repairing loop for services I0331 03:54:44.736319 3904 repair.go:131] Deleted 0 stale service LBs I0331 03:54:44.736339 3904 repair.go:137] Deleted 0 stale Chassis Template Vars I0331 03:54:44.736355 3904 repair.go:62] Finished repairing loop for services: 133.515µs I0331 03:54:44.736382 3904 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0331 03:54:44.736393 3904 services_controller.go:254] Starting workers for network=default I0331 03:54:44.736810 3904 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-lk8gr] creating logical port openshift-multus_network-metrics-daemon-lk8gr for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:44.736897 3904 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-fsh2n] creating logical port openshift-network-diagnostics_network-check-target-fsh2n for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:44.736970 3904 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-lk8gr I0331 03:54:44.737396 3904 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-fsh2n I0331 03:54:44.753523 3904 pod.go:62] [openshift-network-diagnostics/network-check-target-fsh2n] pod update took 16.563714ms I0331 03:54:44.753542 3904 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-fsh2n] addLogicalPort annotation time took 16.584883ms I0331 03:54:44.754365 3904 pods.go:271] [openshift-network-diagnostics/network-check-target-fsh2n] addLogicalPort took 17.477282ms, libovsdb time 610.754µs I0331 03:54:44.754479 3904 pod.go:62] [openshift-multus/network-metrics-daemon-lk8gr] pod update took 17.589124ms I0331 03:54:44.754493 3904 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-lk8gr] addLogicalPort annotation time took 17.607069ms I0331 03:54:44.755249 3904 pods.go:271] [openshift-multus/network-metrics-daemon-lk8gr] addLogicalPort took 18.450408ms, libovsdb time 562.973µs I0331 03:54:44.755976 3904 admin_network_policy_controller.go:227] Starting controller default-network-controller I0331 03:54:44.756004 3904 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0331 03:54:44.756020 3904 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0331 03:54:44.756024 3904 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0331 03:54:44.756251 3904 repair.go:29] Repairing admin network policies took 219.465µs I0331 03:54:44.756431 3904 repair.go:92] Repairing baseline admin network policies took 163.897µs I0331 03:54:44.756732 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0331 03:54:44.756750 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0331 03:54:44.758042 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0331 03:54:44.758061 3904 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0331 03:54:44.758079 3904 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0331 03:54:44.760233 3904 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0331 03:54:44.760251 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0331 03:54:44.760259 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0331 03:54:44.761800 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0331 03:54:44.761818 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0331 03:54:44.761825 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0331 03:54:44.762325 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0331 03:54:44.762339 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0331 03:54:44.762689 3904 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0331 03:54:44.762703 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:54:44.763417 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0331 03:54:44.763436 3904 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0331 03:54:44.763446 3904 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0331 03:54:44.764745 3904 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0331 03:54:44.764764 3904 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0331 03:54:44.764774 3904 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0331 03:54:44.765693 3904 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0331 03:54:44.765711 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0331 03:54:44.765718 3904 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0331 03:54:44.767154 3904 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0331 03:54:44.767174 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0331 03:54:44.767186 3904 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0331 03:54:44.767649 3904 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0331 03:54:44.767667 3904 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0331 03:54:44.767676 3904 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0331 03:54:44.769600 3904 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0331 03:54:44.769618 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0331 03:54:44.769627 3904 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0331 03:54:44.770286 3904 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0331 03:54:44.770303 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0331 03:54:44.770309 3904 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0331 03:54:44.770596 3904 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0331 03:54:44.770617 3904 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0331 03:54:44.770631 3904 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0331 03:54:44.772530 3904 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0331 03:54:44.772549 3904 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0331 03:54:44.772555 3904 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0331 03:54:44.772821 3904 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0331 03:54:44.772838 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0331 03:54:44.772847 3904 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0331 03:54:44.773328 3904 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0331 03:54:44.773349 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0331 03:54:44.773357 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0331 03:54:44.773806 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:54:44.773823 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0331 03:54:44.773830 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0331 03:54:44.774356 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:54:44.774373 3904 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0331 03:54:44.774381 3904 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0331 03:54:44.775178 3904 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0331 03:54:44.775210 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0331 03:54:44.775217 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0331 03:54:44.775640 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0331 03:54:44.775659 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0331 03:54:44.775666 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0331 03:54:44.776833 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0331 03:54:44.776849 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0331 03:54:44.776871 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:54:44.777549 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0331 03:54:44.777565 3904 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0331 03:54:44.777573 3904 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0331 03:54:44.778065 3904 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0331 03:54:44.778082 3904 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0331 03:54:44.778092 3904 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0331 03:54:44.779079 3904 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0331 03:54:44.779096 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0331 03:54:44.779104 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0331 03:54:44.779616 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0331 03:54:44.779631 3904 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0331 03:54:44.779638 3904 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0331 03:54:44.780126 3904 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0331 03:54:44.780144 3904 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0331 03:54:44.780152 3904 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0331 03:54:44.781161 3904 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0331 03:54:44.781180 3904 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0331 03:54:44.781188 3904 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0331 03:54:44.781671 3904 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0331 03:54:44.781686 3904 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0331 03:54:44.781695 3904 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0331 03:54:44.782865 3904 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0331 03:54:44.782884 3904 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0331 03:54:44.782896 3904 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0331 03:54:44.784111 3904 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0331 03:54:44.787928 3904 egressfirewall.go:319] Starting EgressFirewall controller I0331 03:54:44.787956 3904 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0331 03:54:44.787989 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0331 03:54:44.788002 3904 controller.go:133] Adding controller egress-firewall-controller event handlers I0331 03:54:44.788026 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0331 03:54:44.788038 3904 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0331 03:54:44.788004 3904 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0331 03:54:44.788328 3904 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0331 03:54:44.788343 3904 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0331 03:54:44.788352 3904 egressqos.go:193] Setting up event handlers for EgressQoS I0331 03:54:44.788384 3904 egressfirewall.go:628] Syncing node "ip-10-0-2-188.ec2.internal" for egress firewall I0331 03:54:44.788510 3904 egressqos.go:245] Starting EgressQoS Controller I0331 03:54:44.788521 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0331 03:54:44.788536 3904 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0331 03:54:44.788542 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0331 03:54:44.788546 3904 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0331 03:54:44.788551 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0331 03:54:44.788555 3904 shared_informer.go:356] "Caches are synced" controller="egressqos" I0331 03:54:44.788559 3904 egressqos.go:259] Repairing EgressQoSes I0331 03:54:44.788563 3904 egressqos.go:399] Starting repairing loop for egressqos I0331 03:54:44.788636 3904 egressqos.go:401] Finished repairing loop for egressqos: 73.497µs I0331 03:54:44.788656 3904 egressservice_zone.go:132] Setting up event handlers for Egress Services I0331 03:54:44.788689 3904 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-188.ec2.internal I0331 03:54:44.788748 3904 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-188.ec2.internal : 61.006µs I0331 03:54:44.788887 3904 egressservice_zone.go:210] Starting Egress Services Controller I0331 03:54:44.788895 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:54:44.788900 3904 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:54:44.788905 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:54:44.788910 3904 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:54:44.788914 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:54:44.788919 3904 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:54:44.788939 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0331 03:54:44.788945 3904 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0331 03:54:44.788949 3904 egressservice_zone.go:228] Repairing Egress Services I0331 03:54:44.789254 3904 master_controller.go:88] Starting Admin Policy Based Route Controller I0331 03:54:44.789263 3904 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:54:44.789318 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-188.ec2.internal I0331 03:54:44.789331 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-188.ec2.internal: 14.523µs I0331 03:54:44.789470 3904 default_network_controller.go:568] Completing all the Watchers took 295.754339ms I0331 03:54:44.789486 3904 default_network_controller.go:572] Starting unidling controllers I0331 03:54:44.789505 3904 unidle.go:46] Registering OVN SB ControllerEvent handler I0331 03:54:44.789513 3904 unidle.go:63] Populating Initial ContollerEvent events I0331 03:54:44.789522 3904 unidle.go:79] Setting up event handlers for services I0331 03:54:44.789635 3904 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:54:45.042726 3904 default_node_network_controller.go:518] Node connection status = connected I0331 03:54:45.542549 3904 default_node_network_controller.go:518] Node connection status = connected I0331 03:54:46.043371 3904 default_node_network_controller.go:518] Node connection status = connected I0331 03:54:46.205071 3904 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:54:46.542751 3904 default_node_network_controller.go:518] Node connection status = connected I0331 03:54:46.558409 3904 bridgeconfig.go:537] Gateway is ready I0331 03:54:46.558426 3904 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0331 03:54:46.568513 3904 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.188. Updating OVN encap IP. I0331 03:54:46.573088 3904 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.188 - it is already configured I0331 03:54:46.573107 3904 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.188/24:{} 172.20.0.1/32:{}]. Updating annotations. I0331 03:54:46.573387 3904 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.188/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-188.ec2.internal","mac-address":"0e:d2:1c:d7:d3:bb","ip-addresses":["10.0.2.188/24"],"ip-address":"10.0.2.188/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:a2479525-91e9-424b-9527-fe29c4ce6529 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.188/24"}] on node ip-10-0-2-188.ec2.internal I0331 03:54:46.588470 3904 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.188 I0331 03:54:46.588582 3904 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-188.ec2.internal I0331 03:54:46.588787 3904 master.go:532] Adding or Updating local node "ip-10-0-2-188.ec2.internal" for network "default" I0331 03:54:46.588823 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-188.ec2.internal I0331 03:54:46.588836 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-188.ec2.internal: 18.546µs I0331 03:54:46.588879 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-188.ec2.internal in network "default" I0331 03:54:46.588922 3904 node_tracker.go:169] Node ip-10-0-2-188.ec2.internal switch + router changed, syncing services in network "default" I0331 03:54:46.588927 3904 egressfirewall.go:628] Syncing node "ip-10-0-2-188.ec2.internal" for egress firewall I0331 03:54:46.588936 3904 services_controller.go:574] Full service sync requested for network=default I0331 03:54:46.593437 3904 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-188.ec2.internal. Updating no re-route policies I0331 03:54:46.593518 3904 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-trkrq I0331 03:54:46.593526 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qn82n I0331 03:54:46.593527 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-188.ec2.internal I0331 03:54:46.593539 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qn82n I0331 03:54:46.593542 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-tvpth I0331 03:54:46.593555 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-2ds5x I0331 03:54:46.593562 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qn82n after 0 failed attempt(s) I0331 03:54:46.593561 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-lk8gr I0331 03:54:46.593545 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-ttfkk I0331 03:54:46.593572 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-sjn7r I0331 03:54:46.593577 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-ttfkk I0331 03:54:46.593580 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-lk8gr I0331 03:54:46.593581 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-scmk5 I0331 03:54:46.593582 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-gbl64 I0331 03:54:46.593590 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-ttfkk after 0 failed attempt(s) I0331 03:54:46.593581 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-fsh2n I0331 03:54:46.593585 3904 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-j64hm I0331 03:54:46.593597 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-gbl64 I0331 03:54:46.593602 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-fsh2n I0331 03:54:46.593604 3904 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-j64hm I0331 03:54:46.593601 3904 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-188.ec2.internal I0331 03:54:46.593609 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-p6z5z I0331 03:54:46.593613 3904 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-188.ec2.internal I0331 03:54:46.593621 3904 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-lk8gr] creating logical port openshift-multus_network-metrics-daemon-lk8gr for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:46.593622 3904 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-j64hm after 0 failed attempt(s) I0331 03:54:46.593625 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-p6z5z I0331 03:54:46.593562 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-tvpth I0331 03:54:46.593637 3904 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-188.ec2.internal after 0 failed attempt(s) I0331 03:54:46.593641 3904 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-fsh2n] creating logical port openshift-network-diagnostics_network-check-target-fsh2n for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:46.593645 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-tvpth after 0 failed attempt(s) I0331 03:54:46.593647 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-p6z5z after 0 failed attempt(s) I0331 03:54:46.593567 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-188.ec2.internal I0331 03:54:46.593670 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-188.ec2.internal after 0 failed attempt(s) I0331 03:54:46.593590 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-scmk5 I0331 03:54:46.593621 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-gbl64 after 0 failed attempt(s) I0331 03:54:46.593696 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-scmk5 after 0 failed attempt(s) I0331 03:54:46.593535 3904 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-trkrq I0331 03:54:46.593568 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-2ds5x I0331 03:54:46.593713 3904 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-trkrq after 0 failed attempt(s) I0331 03:54:46.593717 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-2ds5x after 0 failed attempt(s) I0331 03:54:46.593584 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-sjn7r I0331 03:54:46.593730 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-sjn7r after 0 failed attempt(s) I0331 03:54:46.594358 3904 pods.go:271] [openshift-network-diagnostics/network-check-target-fsh2n] addLogicalPort took 726.86µs, libovsdb time 244.971µs I0331 03:54:46.594370 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-fsh2n after 0 failed attempt(s) I0331 03:54:46.594428 3904 pods.go:271] [openshift-multus/network-metrics-daemon-lk8gr] addLogicalPort took 818.929µs, libovsdb time 250.699µs I0331 03:54:46.594437 3904 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-lk8gr after 0 failed attempt(s) I0331 03:54:46.605808 3904 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0331 03:54:46.616999 3904 iptables.go:108] Creating table: mangle chain: OUTPUT I0331 03:54:46.620496 3904 iptables.go:108] Creating table: nat chain: OUTPUT I0331 03:54:46.638624 3904 iptables.go:108] Creating table: nat chain: PREROUTING I0331 03:54:46.679751 3904 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:54:46.698825 3904 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0331 03:54:46.698957 3904 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0331 03:54:46.699020 3904 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0331 03:54:46.699120 3904 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0331 03:54:46.699166 3904 gateway_init.go:325] Gateway and management port readiness took 2.160345664s I0331 03:54:46.699208 3904 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0331 03:54:46.699247 3904 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0331 03:54:46.699261 3904 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0331 03:54:46.699271 3904 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:54:46.699324 3904 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0331 03:54:46.699414 3904 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 43.394µs I0331 03:54:46.699561 3904 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0331 03:54:46.699589 3904 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 19.047µs I0331 03:54:46.699597 3904 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 37.075µs I0331 03:54:46.699603 3904 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0331 03:54:46.699612 3904 nad_controller.go:173] [node-nad-controller NAD controller]: started I0331 03:54:46.699616 3904 default_node_network_controller.go:898] Starting the default node network controller I0331 03:54:46.699621 3904 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0331 03:54:46.699702 3904 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.852µs I0331 03:54:46.699739 3904 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0331 03:54:46.699748 3904 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0331 03:54:46.699770 3904 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1d580 0xc0002ec230 0xc0002ec2a0 0xc0002ec310 0xc0002ec380 0xc0002ec3f0 0xc0002ec4d0 0xc000b2b7c0 0xc0002ec540 0xc0002ec5b0 0xc000818e60 0xc0064fcfc0 0xc0002ec620 map[0x2f67040:0xc002492300 0x2f67580:0xc002492480 0x2f68000:0xc002252900 0x2f682a0:0xc0024922a0 0x2f687e0:0xc002492360 0x2f68a80:0xc0024923c0 0x2f68d20:0xc002492420 0x2f69500:0xc0024924e0 0x2f697a0:0xc002492600 0x2f93d40:0xc002252720 0x2f940e0:0xc002252780 0x2f94bc0:0xc000b8b980 0x2f96520:0xc001bcc000 0x2f97000:0xc0022528a0 0x2f97e80:0xc000f12e40] 0xc000b20700 0} I0331 03:54:46.699822 3904 gateway.go:151] Starting gateway service sync I0331 03:54:46.700244 3904 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0331 03:54:46.702720 3904 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0331 03:54:46.713026 3904 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0331 03:54:46.714314 3904 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0331 03:54:46.715553 3904 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0331 03:54:46.748830 3904 gateway.go:168] Gateway service sync done. Time taken: 48.994515ms I0331 03:54:46.749392 3904 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32442 I0331 03:54:46.749674 3904 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1d580 0xc0002ec230 0xc0002ec2a0 0xc0002ec310 0xc0002ec380 0xc0002ec3f0 0xc0002ec4d0 0xc000b2b7c0 0xc0002ec540 0xc0002ec5b0 0xc000818e60 0xc0064fcfc0 0xc0002ec620 map[0x2f67040:0xc002492300 0x2f67580:0xc002492480 0x2f68000:0xc002252900 0x2f682a0:0xc0024922a0 0x2f687e0:0xc002492360 0x2f68a80:0xc0024923c0 0x2f68d20:0xc002492420 0x2f69500:0xc0024924e0 0x2f697a0:0xc002492600 0x2f93d40:0xc002252720 0x2f940e0:0xc002252780 0x2f94bc0:0xc000b8b980 0x2f96520:0xc001bcc000 0x2f97000:0xc0022528a0 0x2f97e80:0xc000f12e40] 0xc000b20700 0} I0331 03:54:46.750222 3904 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1d580 0xc0002ec230 0xc0002ec2a0 0xc0002ec310 0xc0002ec380 0xc0002ec3f0 0xc0002ec4d0 0xc000b2b7c0 0xc0002ec540 0xc0002ec5b0 0xc000818e60 0xc0064fcfc0 0xc0002ec620 map[0x2f67040:0xc002492300 0x2f67580:0xc002492480 0x2f68000:0xc002252900 0x2f682a0:0xc0024922a0 0x2f687e0:0xc002492360 0x2f68a80:0xc0024923c0 0x2f68d20:0xc002492420 0x2f69500:0xc0024924e0 0x2f697a0:0xc002492600 0x2f93d40:0xc002252720 0x2f940e0:0xc002252780 0x2f94bc0:0xc000b8b980 0x2f96520:0xc001bcc000 0x2f97000:0xc0022528a0 0x2f97e80:0xc000f12e40] 0xc000b20700 0} I0331 03:54:46.750284 3904 gateway.go:519] Reconciling gateway with updates I0331 03:54:46.755346 3904 gateway.go:326] Spawning Conntrack Rule Check Thread I0331 03:54:46.755429 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:54:46.755439 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:54:46.755447 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:54:46.755450 3904 default_node_network_controller.go:943] Gateway and management port readiness took 55.693491ms I0331 03:54:46.755453 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:54:46.755461 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:54:46.755451 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:54:46.755469 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:54:46.755481 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:54:46.755487 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:54:46.755499 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:54:46.755508 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:54:46.755497 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:54:46.755520 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:54:46.755522 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:54:46.755543 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:54:46.755553 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:54:46.755554 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:54:46.755558 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:54:46.755564 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:54:46.755564 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:54:46.755570 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:54:46.755573 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:54:46.755575 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:54:46.755577 3904 port_claim.go:71] Svc openshift-ingress/router-default: port 30231 is already open, no action needed I0331 03:54:46.755579 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:54:46.755578 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:54:46.755586 3904 port_claim.go:71] Svc openshift-ingress/router-default: port 32345 is already open, no action needed I0331 03:54:46.755587 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:54:46.755592 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:54:46.755596 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:54:46.755504 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:54:46.755598 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:54:46.755526 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:54:46.755613 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:54:46.755613 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:54:46.755622 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:54:46.755628 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:54:46.755634 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:54:46.755640 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:54:46.755642 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:54:46.755644 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:54:46.755518 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:54:46.755567 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:54:46.755656 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:54:46.755657 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:54:46.755660 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:54:46.755566 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:54:46.755511 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:54:46.755519 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:54:46.755671 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:54:46.755659 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:54:46.755684 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:54:46.755630 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:54:46.755693 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:54:46.755701 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:54:46.755576 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:54:46.755588 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:54:46.755713 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:54:46.755717 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:54:46.755623 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:54:46.755729 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:54:46.755727 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:54:46.755737 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:54:46.755740 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:54:46.755735 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:54:46.755743 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:54:46.755747 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:54:46.755753 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:54:46.755757 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:54:46.755760 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:54:46.755761 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:54:46.755742 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:54:46.755693 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:54:46.755760 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:54:46.755583 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:54:46.755800 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:54:46.755546 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:54:46.755815 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:54:46.755466 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:54:46.755646 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:54:46.755714 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:54:46.755859 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:54:46.755554 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:54:46.755860 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:54:46.755870 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:54:46.755876 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:54:46.755875 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:54:46.755883 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:54:46.755671 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:54:46.755890 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:54:46.755701 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:54:46.755608 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:54:46.755900 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:54:46.755607 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:54:46.755907 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:54:46.755912 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:54:46.755925 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:54:46.755439 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:54:46.755977 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:54:46.755453 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:54:46.755619 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:54:46.756019 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:54:46.755665 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:54:46.755649 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:54:46.755516 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:54:46.755725 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:54:46.755727 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:54:46.755704 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:54:46.755742 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:54:46.756142 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:54:46.756120 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:54:46.756189 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:54:46.755604 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:54:46.757021 3904 node_ip_handler_linux.go:160] Node IP manager is running W0331 03:54:46.794905 3904 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:54:46.795107 3904 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:54:46.795279 3904 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0331 03:54:46.803362 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:54:46.803384 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:54:46.803393 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:54:46.803401 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:54:46.803407 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:54:46.803414 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:54:46.803419 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:54:46.840667 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:54:46.840692 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:54:46.840701 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:54:46.840707 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:54:46.840715 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:54:46.840720 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:54:46.840726 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:54:46.840731 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:54:46.840738 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:54:46.840744 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:54:46.840749 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:54:46.840754 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:54:46.840761 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:54:46.840766 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:54:48.195889 3904 default_node_network_controller.go:1364] Starting node controller node sync W0331 03:54:48.195893 3904 egressip_healthcheck.go:76] Health checking using insecure connection I0331 03:54:48.195972 3904 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0331 03:54:48.207896 3904 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.985066ms I0331 03:54:48.208154 3904 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0331 03:54:48.208511 3904 egressservice_node.go:93] Setting up event handlers for Egress Services I0331 03:54:48.208771 3904 egressservice_node.go:183] Starting Egress Services Controller I0331 03:54:48.208812 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:54:48.208826 3904 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:54:48.208842 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:54:48.208850 3904 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:54:48.208857 3904 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:54:48.208865 3904 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:54:48.208871 3904 egressservice_node.go:197] Repairing Egress Services I0331 03:54:48.219568 3904 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0331 03:54:48.219580 3904 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:54:48.219855 3904 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0331 03:54:48.219870 3904 link_network_manager.go:117] Link manager is running I0331 03:54:48.219883 3904 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0331 03:54:48.219987 3904 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0331 03:54:48.220504 3904 vrf_manager.go:128] VRF manager is running I0331 03:54:48.220585 3904 node_controller_manager.go:437] Removing flows to drop GARP I0331 03:54:48.220594 3904 gateway.go:519] Reconciling gateway with updates I0331 03:54:48.250305 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:54:48.250318 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:54:48.250332 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:54:48.250335 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:54:48.250343 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:54:48.250346 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:54:48.250392 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:54:48.250410 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:54:48.250421 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:54:48.250322 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:54:48.250430 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:54:48.250495 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:54:48.250500 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:54:48.250507 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:54:48.250509 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:54:48.250494 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:54:48.250519 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:54:48.250523 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:54:48.250526 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:54:48.250530 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:54:48.250530 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:54:48.250413 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:54:48.250517 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:54:48.250545 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:54:48.250498 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:54:48.250554 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:54:48.250558 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:54:48.250567 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:54:48.250568 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:54:48.250579 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:54:48.250584 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:54:48.250588 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:54:48.250529 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:54:48.250604 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:54:48.250615 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:54:48.250505 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:54:48.250624 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:54:48.250417 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:54:48.250634 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:54:48.250641 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:54:48.250644 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:54:48.250649 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:54:48.250654 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:54:48.250658 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:54:48.250660 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:54:48.250666 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:54:48.250557 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:54:48.250434 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:54:48.250532 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:54:48.250712 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:54:48.250446 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:54:48.250735 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:54:48.250747 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:54:48.250752 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:54:48.250757 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:54:48.250757 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:54:48.250762 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:54:48.250771 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:54:48.250760 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:54:48.250888 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:54:48.250730 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:54:48.250896 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:54:48.250906 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:54:48.250911 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:54:48.250914 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:54:48.250457 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:54:48.251003 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:54:48.250469 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:54:48.251072 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:54:48.251081 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:54:48.250474 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:54:48.251129 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:54:48.250470 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:54:48.251172 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:54:48.250481 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:54:48.251224 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:54:48.251229 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:54:48.250477 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:54:48.251239 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:54:48.251242 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:54:48.250484 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:54:48.251248 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:54:48.250488 3904 port_claim.go:71] Svc openshift-ingress/router-default: port 30231 is already open, no action needed I0331 03:54:48.251291 3904 port_claim.go:71] Svc openshift-ingress/router-default: port 32345 is already open, no action needed I0331 03:54:48.250492 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:54:48.251336 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:54:48.251341 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:54:48.250497 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:54:48.251349 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:54:48.250515 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:54:48.251372 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:54:48.250517 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:54:48.250413 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:54:48.251399 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:54:48.250441 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:54:48.250717 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:54:48.251468 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:54:48.250441 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:54:48.251504 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:54:48.250457 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:54:48.251549 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:54:48.251556 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:54:48.250461 3904 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:54:48.251569 3904 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:54:48.297565 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:54:48.297594 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:54:48.297605 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:54:48.297614 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:54:48.297620 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:54:48.297628 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:54:48.297639 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:54:48.297648 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:54:48.297653 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:54:48.297662 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:54:48.336048 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:54:48.336071 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:54:48.336079 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:54:48.336085 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:54:48.336093 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:54:48.336099 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:54:48.336123 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:54:48.336144 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:54:48.336153 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:54:48.336161 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:54:48.336169 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:54:48.336177 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:54:48.373918 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:54:48.373928 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:54:48.373945 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:54:48.373952 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:54:48.373960 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:54:48.373966 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:54:48.373972 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:54:48.373979 3904 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:54:48.817934 3904 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-trkrq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:54:48.818211 3904 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-trkrq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:54:49.819295 3904 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-trkrq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:54:49.819328 3904 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-trkrq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:54:50.466251 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-188.ec2.internal I0331 03:54:50.466286 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-188.ec2.internal: 48.521µs I0331 03:54:50.492584 3904 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-68db74b777-7xc9h] creating logical port openshift-image-registry_image-registry-68db74b777-7xc9h for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.492789 3904 kube.go:256] Updating pod openshift-image-registry/image-registry-68db74b777-7xc9h I0331 03:54:50.517059 3904 pod.go:62] [openshift-image-registry/image-registry-68db74b777-7xc9h] pod update took 24.312322ms I0331 03:54:50.517080 3904 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-68db74b777-7xc9h] addLogicalPort annotation time took 24.338515ms I0331 03:54:50.518280 3904 pods.go:271] [openshift-image-registry/image-registry-68db74b777-7xc9h] addLogicalPort took 25.715993ms, libovsdb time 906.421µs I0331 03:54:50.540453 3904 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-zcl7p] creating logical port openshift-dns_dns-default-zcl7p for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.540564 3904 kube.go:256] Updating pod openshift-dns/dns-default-zcl7p I0331 03:54:50.541581 3904 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-kf8bj] creating logical port openshift-insights_insights-operator-5577457d66-kf8bj for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.541683 3904 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-x5k7f for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.541689 3904 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-kf8bj I0331 03:54:50.541837 3904 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f I0331 03:54:50.542071 3904 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.542145 3904 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l I0331 03:54:50.544290 3904 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6fb4878899-lt98g] creating logical port openshift-ingress_router-default-6fb4878899-lt98g for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.544426 3904 kube.go:256] Updating pod openshift-ingress/router-default-6fb4878899-lt98g I0331 03:54:50.544472 3904 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-7fscs for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.544488 3904 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-ffc7ccb6f-vf8sr] creating logical port openshift-ingress_router-default-ffc7ccb6f-vf8sr for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.544613 3904 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs I0331 03:54:50.544727 3904 kube.go:256] Updating pod openshift-ingress/router-default-ffc7ccb6f-vf8sr I0331 03:54:50.566163 3904 pod.go:62] [openshift-dns/dns-default-zcl7p] pod update took 25.620115ms I0331 03:54:50.566183 3904 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-zcl7p] addLogicalPort annotation time took 25.645247ms I0331 03:54:50.566385 3904 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l] pod update took 24.254144ms I0331 03:54:50.566407 3904 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l] addLogicalPort annotation time took 24.281548ms I0331 03:54:50.566417 3904 pod.go:62] [openshift-insights/insights-operator-5577457d66-kf8bj] pod update took 24.753727ms I0331 03:54:50.566450 3904 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-kf8bj] addLogicalPort annotation time took 24.770481ms I0331 03:54:50.566877 3904 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-8zl84] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-8zl84 for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.566969 3904 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-8zl84 I0331 03:54:50.567460 3904 pods.go:271] [openshift-dns/dns-default-zcl7p] addLogicalPort took 27.024216ms, libovsdb time 993.199µs I0331 03:54:50.567778 3904 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l] addLogicalPort took 25.714384ms, libovsdb time 1.118367ms I0331 03:54:50.568177 3904 pods.go:271] [openshift-insights/insights-operator-5577457d66-kf8bj] addLogicalPort took 26.601365ms, libovsdb time 1.360333ms I0331 03:54:50.569335 3904 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-t7fz8 for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.569341 3904 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f] pod update took 27.523689ms I0331 03:54:50.569464 3904 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8 I0331 03:54:50.569480 3904 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f] addLogicalPort annotation time took 27.659956ms I0331 03:54:50.569638 3904 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-h9wgk] creating logical port openshift-ingress-canary_ingress-canary-h9wgk for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.569716 3904 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-h9wgk I0331 03:54:50.569952 3904 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-8c69g for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.570021 3904 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g I0331 03:54:50.570568 3904 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f] addLogicalPort took 28.898749ms, libovsdb time 812.575µs I0331 03:54:50.575458 3904 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs] pod update took 30.878511ms I0331 03:54:50.575475 3904 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-wctw7] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-wctw7 for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.575484 3904 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs] addLogicalPort annotation time took 30.903582ms I0331 03:54:50.575545 3904 pod.go:62] [openshift-ingress/router-default-6fb4878899-lt98g] pod update took 31.14254ms I0331 03:54:50.575563 3904 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-6fb4878899-lt98g] addLogicalPort annotation time took 31.16511ms I0331 03:54:50.575591 3904 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-wctw7 I0331 03:54:50.576591 3904 pods.go:271] [openshift-ingress/router-default-6fb4878899-lt98g] addLogicalPort took 32.311192ms, libovsdb time 788.088µs I0331 03:54:50.576635 3904 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-bgngr for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.576695 3904 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr I0331 03:54:50.577118 3904 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs] addLogicalPort took 32.656751ms, libovsdb time 781.099µs I0331 03:54:50.582539 3904 pod.go:62] [openshift-ingress/router-default-ffc7ccb6f-vf8sr] pod update took 37.831125ms I0331 03:54:50.582557 3904 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-ffc7ccb6f-vf8sr] addLogicalPort annotation time took 37.849825ms I0331 03:54:50.583513 3904 pods.go:271] [openshift-ingress/router-default-ffc7ccb6f-vf8sr] addLogicalPort took 39.033517ms, libovsdb time 697.563µs I0331 03:54:50.593534 3904 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-8zl84] pod update took 26.590142ms I0331 03:54:50.593557 3904 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-8zl84] addLogicalPort annotation time took 26.616154ms I0331 03:54:50.593862 3904 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8] pod update took 24.41928ms I0331 03:54:50.593884 3904 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8] addLogicalPort annotation time took 24.443983ms I0331 03:54:50.594110 3904 pod.go:62] [openshift-ingress-canary/ingress-canary-h9wgk] pod update took 24.409519ms I0331 03:54:50.594127 3904 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-h9wgk] addLogicalPort annotation time took 24.428289ms I0331 03:54:50.594573 3904 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-8zl84] addLogicalPort took 27.710604ms, libovsdb time 721.582µs I0331 03:54:50.595043 3904 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8] addLogicalPort took 25.716557ms, libovsdb time 851.749µs I0331 03:54:50.595098 3904 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-xprbl] creating logical port openshift-console-operator_console-operator-8777bd9bb-xprbl for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.595184 3904 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-xprbl I0331 03:54:50.595458 3904 pods.go:271] [openshift-ingress-canary/ingress-canary-h9wgk] addLogicalPort took 25.828044ms, libovsdb time 1.093154ms I0331 03:54:50.595507 3904 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-4djw8 for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:50.595553 3904 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8 I0331 03:54:50.611219 3904 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g] pod update took 41.220139ms I0331 03:54:50.611248 3904 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-wctw7] pod update took 35.725115ms I0331 03:54:50.611253 3904 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g] addLogicalPort annotation time took 41.252626ms I0331 03:54:50.611264 3904 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-wctw7] addLogicalPort annotation time took 35.742037ms I0331 03:54:50.611265 3904 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr] pod update took 34.589061ms I0331 03:54:50.611282 3904 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr] addLogicalPort annotation time took 34.605636ms I0331 03:54:50.612251 3904 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-wctw7] addLogicalPort took 36.787583ms, libovsdb time 666.823µs I0331 03:54:50.612777 3904 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g] addLogicalPort took 42.832088ms, libovsdb time 1.178679ms I0331 03:54:50.613013 3904 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr] addLogicalPort took 36.382446ms, libovsdb time 1.407599ms I0331 03:54:50.627579 3904 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8] pod update took 32.033554ms I0331 03:54:50.627660 3904 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8] addLogicalPort annotation time took 32.11508ms I0331 03:54:50.629003 3904 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8] addLogicalPort took 33.498409ms, libovsdb time 954.524µs I0331 03:54:50.637143 3904 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-xprbl] pod update took 41.999571ms I0331 03:54:50.637165 3904 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-xprbl] addLogicalPort annotation time took 42.029422ms I0331 03:54:50.638349 3904 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-xprbl] addLogicalPort took 43.255589ms, libovsdb time 816.174µs I0331 03:54:50.948316 3904 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l 286ac6ac64dbd2db85b42e3058b1b19efdfd62c837973b8881f9828a7114efc9 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l 286ac6ac64dbd2db85b42e3058b1b19efdfd62c837973b8881f9828a7114efc9 network default NAD default] I0331 03:54:50.962495 3904 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-xprbl 42d5f49c5fe3737939b7e227a3cfbba696a0ee9304a72a4d47c002a6ddff0947 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-xprbl 42d5f49c5fe3737939b7e227a3cfbba696a0ee9304a72a4d47c002a6ddff0947 network default NAD default] I0331 03:54:50.967242 3904 cni.go:403] [openshift-ingress-canary/ingress-canary-h9wgk 1982ed3080362f4840a7050d8f763ad3298db12fd1edf8984ac962ed5c856c8b network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-h9wgk 1982ed3080362f4840a7050d8f763ad3298db12fd1edf8984ac962ed5c856c8b network default NAD default] I0331 03:54:50.978814 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l, hostIfaceName: 286ac6ac64dbd2d, network: default, NAD default, SandboxID: "286ac6ac64dbd2db85b42e3058b1b19efdfd62c837973b8881f9828a7114efc9", PCI device ID: , UID: "53c41072-257b-4cdb-aa6c-db7cfd1dabd9", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0331 03:54:50.993287 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-xprbl, hostIfaceName: 42d5f49c5fe3737, network: default, NAD default, SandboxID: "42d5f49c5fe3737939b7e227a3cfbba696a0ee9304a72a4d47c002a6ddff0947", PCI device ID: , UID: "19b6bbce-3d4c-49f6-8830-7d5c3109be47", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0331 03:54:50.995401 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-h9wgk, hostIfaceName: 1982ed3080362f4, network: default, NAD default, SandboxID: "1982ed3080362f4840a7050d8f763ad3298db12fd1edf8984ac962ed5c856c8b", PCI device ID: , UID: "a8dcbbb6-02b6-4b9c-9092-e1cf7bf7073d", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0331 03:54:50.997910 3904 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g 4a3e5fcb32a12dda4b39eafec32b9e8254136d50b4bea657509a2ada369f5761 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g 4a3e5fcb32a12dda4b39eafec32b9e8254136d50b4bea657509a2ada369f5761 network default NAD default] I0331 03:54:51.014409 3904 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr 12e5d3113a1c194de2337ec99cd8af94bd23b6e486bf7fec84f0efec0e17c629 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr 12e5d3113a1c194de2337ec99cd8af94bd23b6e486bf7fec84f0efec0e17c629 network default NAD default] I0331 03:54:51.015891 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-8c69g, hostIfaceName: 4a3e5fcb32a12dd, network: default, NAD default, SandboxID: "4a3e5fcb32a12dda4b39eafec32b9e8254136d50b4bea657509a2ada369f5761", PCI device ID: , UID: "2f372c0f-ffd7-42b8-a132-a39e975e7b79", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0331 03:54:51.031149 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-bgngr, hostIfaceName: 12e5d3113a1c194, network: default, NAD default, SandboxID: "12e5d3113a1c194de2337ec99cd8af94bd23b6e486bf7fec84f0efec0e17c629", PCI device ID: , UID: "8627a84e-629e-4a6c-89cc-78fee4908a40", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0331 03:54:51.033534 3904 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l 286ac6ac64dbd2db85b42e3058b1b19efdfd62c837973b8881f9828a7114efc9 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-bhr4l 286ac6ac64dbd2db85b42e3058b1b19efdfd62c837973b8881f9828a7114efc9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:97:0b:d7:5c:aa\",\"name\":\"286ac6ac64dbd2d\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1dbbec7f-d213-42d3-ac36-da2c19b456ca\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.053421 3904 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-xprbl 42d5f49c5fe3737939b7e227a3cfbba696a0ee9304a72a4d47c002a6ddff0947 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-xprbl 42d5f49c5fe3737939b7e227a3cfbba696a0ee9304a72a4d47c002a6ddff0947 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:fc:45:32:2f:15\",\"name\":\"42d5f49c5fe3737\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/28ae32b5-576e-440c-821a-0dd98b66e2e3\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.054699 3904 cni.go:424] [openshift-ingress-canary/ingress-canary-h9wgk 1982ed3080362f4840a7050d8f763ad3298db12fd1edf8984ac962ed5c856c8b network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-h9wgk 1982ed3080362f4840a7050d8f763ad3298db12fd1edf8984ac962ed5c856c8b network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:64:1c:f9:d2:22\",\"name\":\"1982ed3080362f4\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ae89171-026d-42b3-b67a-949e0d3928f2\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.064352 3904 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g 4a3e5fcb32a12dda4b39eafec32b9e8254136d50b4bea657509a2ada369f5761 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8c69g 4a3e5fcb32a12dda4b39eafec32b9e8254136d50b4bea657509a2ada369f5761 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:cf:2a:d9:2f:f0\",\"name\":\"4a3e5fcb32a12dd\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/04acdd7e-65ab-4cf6-882c-cf11af36e6e6\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.074783 3904 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr 12e5d3113a1c194de2337ec99cd8af94bd23b6e486bf7fec84f0efec0e17c629 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-bgngr 12e5d3113a1c194de2337ec99cd8af94bd23b6e486bf7fec84f0efec0e17c629 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:7b:87:35:ef:96\",\"name\":\"12e5d3113a1c194\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/184b5893-2789-4d7b-8256-f8bba0d5f548\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.210870 3904 cni.go:403] [openshift-insights/insights-operator-5577457d66-kf8bj 85599d3d39d7a86489f0931f7738610e0ff3bc9ad8afd02553bf6993b8030cbe network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-kf8bj 85599d3d39d7a86489f0931f7738610e0ff3bc9ad8afd02553bf6993b8030cbe network default NAD default] I0331 03:54:51.213927 3904 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f 164bf4e8785f68c0870ad589162da161f24c4fefe02ce61e7cd8923f632adcfe network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f 164bf4e8785f68c0870ad589162da161f24c4fefe02ce61e7cd8923f632adcfe network default NAD default] I0331 03:54:51.224618 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-kf8bj, hostIfaceName: 85599d3d39d7a86, network: default, NAD default, SandboxID: "85599d3d39d7a86489f0931f7738610e0ff3bc9ad8afd02553bf6993b8030cbe", PCI device ID: , UID: "8c953f23-b691-4bd5-ab88-4eb61d7b75a3", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0331 03:54:51.226398 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-x5k7f, hostIfaceName: 164bf4e8785f68c, network: default, NAD default, SandboxID: "164bf4e8785f68c0870ad589162da161f24c4fefe02ce61e7cd8923f632adcfe", PCI device ID: , UID: "b15f7ff6-7e5a-44f0-b1b8-0651f3bf39aa", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0331 03:54:51.254717 3904 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8 de3b86be8d378be0196b4a3c34928de1218422f31ce59b1e43f35b5187115cc2 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8 de3b86be8d378be0196b4a3c34928de1218422f31ce59b1e43f35b5187115cc2 network default NAD default] I0331 03:54:51.265509 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-t7fz8, hostIfaceName: de3b86be8d378be, network: default, NAD default, SandboxID: "de3b86be8d378be0196b4a3c34928de1218422f31ce59b1e43f35b5187115cc2", PCI device ID: , UID: "c2616122-f6f7-42a5-b98c-ed04602848d4", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0331 03:54:51.267845 3904 cni.go:424] [openshift-insights/insights-operator-5577457d66-kf8bj 85599d3d39d7a86489f0931f7738610e0ff3bc9ad8afd02553bf6993b8030cbe network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-kf8bj 85599d3d39d7a86489f0931f7738610e0ff3bc9ad8afd02553bf6993b8030cbe network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:fc:03:5a:90:6d\",\"name\":\"85599d3d39d7a86\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2bc2a39e-e0e2-4674-8ec6-50d641b6fd08\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.274993 3904 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f 164bf4e8785f68c0870ad589162da161f24c4fefe02ce61e7cd8923f632adcfe network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x5k7f 164bf4e8785f68c0870ad589162da161f24c4fefe02ce61e7cd8923f632adcfe network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:53:75:39:91:e9\",\"name\":\"164bf4e8785f68c\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e76b93d-5d4f-4a5b-bcfc-2b1ef5e1c678\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:51.307178 3904 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8 de3b86be8d378be0196b4a3c34928de1218422f31ce59b1e43f35b5187115cc2 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t7fz8 de3b86be8d378be0196b4a3c34928de1218422f31ce59b1e43f35b5187115cc2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:f9:73:4f:d3:1c\",\"name\":\"de3b86be8d378be\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6e34e2f9-e818-4714-83c2-5def8cf1e3dc\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:52.293484 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:54:52.293573 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 98.357µs I0331 03:54:52.293492 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:54:52.293638 3904 node_tracker.go:220] Node ip-10-0-0-60.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0331 03:54:52.293537 3904 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall E0331 03:54:52.293652 3904 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-60.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0331 03:54:52.293671 3904 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-60.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-60.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-60.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-60.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-60.ec2.internal, error parsing annotation for node ip-10-0-0-60.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0331 03:54:52.307699 3904 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-60.ec2.internal annotation: unexpected end of JSON input I0331 03:54:52.307922 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0331 03:54:52.307991 3904 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-60.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0331 03:54:52.323021 3904 egressfirewall.go:628] Syncing node "ip-10-0-2-188.ec2.internal" for egress firewall I0331 03:54:52.323235 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-188.ec2.internal I0331 03:54:52.323265 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-188.ec2.internal: 32.987µs E0331 03:54:52.323322 3904 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-60.ec2.internal annotation: unexpected end of JSON input I0331 03:54:52.323369 3904 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall I0331 03:54:52.323370 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0331 03:54:52.323413 3904 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-60.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0331 03:54:52.323483 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:54:52.323497 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 18.588µs E0331 03:54:52.401183 3904 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-60.ec2.internal annotation: unexpected end of JSON input E0331 03:54:52.401343 3904 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-60.ec2.internal annotation: unexpected end of JSON input I0331 03:54:52.401370 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0331 03:54:52.401446 3904 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-60.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-60.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-60.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-60.ec2.internal" I0331 03:54:52.413504 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0331 03:54:52.491379 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:54:52.491463 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 91.113µs I0331 03:54:52.491763 3904 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall I0331 03:54:52.491895 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:54:52.506532 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:54:52.690916 3904 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-r8j8m] creating logical port kube-system_global-pull-secret-syncer-r8j8m for pod on switch ip-10-0-2-188.ec2.internal I0331 03:54:52.691052 3904 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-r8j8m I0331 03:54:52.703767 3904 pod.go:62] [kube-system/global-pull-secret-syncer-r8j8m] pod update took 12.749705ms I0331 03:54:52.703792 3904 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-r8j8m] addLogicalPort annotation time took 12.781382ms I0331 03:54:52.705304 3904 pods.go:271] [kube-system/global-pull-secret-syncer-r8j8m] addLogicalPort took 14.40669ms, libovsdb time 1.052497ms I0331 03:54:52.760517 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:54:53.615298 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:54:58.224550 3904 cni.go:403] [openshift-network-diagnostics/network-check-target-fsh2n 644ec0f8edbaa80dcf717f64a5cc8ce1fdb790291eda5951a0a084858c3a3129 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-fsh2n 644ec0f8edbaa80dcf717f64a5cc8ce1fdb790291eda5951a0a084858c3a3129 network default NAD default] I0331 03:54:58.234098 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-fsh2n, hostIfaceName: 644ec0f8edbaa80, network: default, NAD default, SandboxID: "644ec0f8edbaa80dcf717f64a5cc8ce1fdb790291eda5951a0a084858c3a3129", PCI device ID: , UID: "62e6de9a-ae66-4235-8571-54503f166c1b", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0331 03:54:58.277702 3904 cni.go:424] [openshift-network-diagnostics/network-check-target-fsh2n 644ec0f8edbaa80dcf717f64a5cc8ce1fdb790291eda5951a0a084858c3a3129 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-fsh2n 644ec0f8edbaa80dcf717f64a5cc8ce1fdb790291eda5951a0a084858c3a3129 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:74:2d:22:5f:4c\",\"name\":\"644ec0f8edbaa80\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c32d6bcc-5e53-4091-8627-b7e17e79a4c2\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:58.425271 3904 cni.go:403] [kube-system/global-pull-secret-syncer-r8j8m 72810a227718a0ea679ec94f69de30f08a9631eda375f0869c45db523d6cdbd4 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-r8j8m 72810a227718a0ea679ec94f69de30f08a9631eda375f0869c45db523d6cdbd4 network default NAD default] I0331 03:54:58.436369 3904 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-r8j8m, hostIfaceName: 72810a227718a0e, network: default, NAD default, SandboxID: "72810a227718a0ea679ec94f69de30f08a9631eda375f0869c45db523d6cdbd4", PCI device ID: , UID: "278459e3-cccc-4a87-b142-3c31a11f7b9d", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0331 03:54:58.480070 3904 cni.go:424] [kube-system/global-pull-secret-syncer-r8j8m 72810a227718a0ea679ec94f69de30f08a9631eda375f0869c45db523d6cdbd4 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-r8j8m 72810a227718a0ea679ec94f69de30f08a9631eda375f0869c45db523d6cdbd4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:67:42:28:df:f4\",\"name\":\"72810a227718a0e\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f634b4bf-39d6-463e-9915-48d24fc51994\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:54:59.865284 3904 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:54:59.920591 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0331 03:55:00.068806 3904 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:55:01.299739 3904 namespace.go:98] [openshift-service-ca] adding namespace I0331 03:55:01.300749 3904 namespace.go:102] [openshift-service-ca] adding namespace took 986.545µs I0331 03:55:01.315926 3904 namespace.go:142] [openshift-service-ca] updating namespace I0331 03:55:01.391939 3904 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0331 03:55:01.392977 3904 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.012708ms I0331 03:55:01.408674 3904 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0331 03:55:01.606474 3904 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-vxqnr for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:01.606578 3904 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr I0331 03:55:01.622742 3904 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr] pod update took 16.187562ms I0331 03:55:01.622769 3904 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr] addLogicalPort annotation time took 16.220433ms I0331 03:55:01.624172 3904 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr] addLogicalPort took 17.711047ms, libovsdb time 1.021657ms I0331 03:55:02.400949 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:02.965754 3904 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-x6hq2] creating logical port openshift-service-ca_service-ca-f59fc589b-x6hq2 for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:02.965854 3904 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-x6hq2 I0331 03:55:02.969844 3904 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr 76d29b73927bd7dd82face4df5708a49b583c2a9211006a4686e438cd5ae1a22 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr 76d29b73927bd7dd82face4df5708a49b583c2a9211006a4686e438cd5ae1a22 network default NAD default] I0331 03:55:02.979137 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-vxqnr, hostIfaceName: 76d29b73927bd7d, network: default, NAD default, SandboxID: "76d29b73927bd7dd82face4df5708a49b583c2a9211006a4686e438cd5ae1a22", PCI device ID: , UID: "78de1e9a-50c5-4353-bd21-6b1886194315", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0331 03:55:02.979956 3904 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-x6hq2] pod update took 14.126783ms I0331 03:55:02.979979 3904 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-x6hq2] addLogicalPort annotation time took 14.151389ms I0331 03:55:02.981183 3904 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-x6hq2] addLogicalPort took 15.452155ms, libovsdb time 789.385µs I0331 03:55:03.025079 3904 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr 76d29b73927bd7dd82face4df5708a49b583c2a9211006a4686e438cd5ae1a22 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-vxqnr 76d29b73927bd7dd82face4df5708a49b583c2a9211006a4686e438cd5ae1a22 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:a4:09:cb:1c:5b\",\"name\":\"76d29b73927bd7d\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6d41596e-ee51-4466-be20-0f0815c9a0ad\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:03.370857 3904 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-x6hq2 3e809162a393688cfe195be8481a870926a6c61f460e10d855bbf7e8d1f797d0 network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-x6hq2 3e809162a393688cfe195be8481a870926a6c61f460e10d855bbf7e8d1f797d0 network default NAD default] I0331 03:55:03.380930 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-x6hq2, hostIfaceName: 3e809162a393688, network: default, NAD default, SandboxID: "3e809162a393688cfe195be8481a870926a6c61f460e10d855bbf7e8d1f797d0", PCI device ID: , UID: "bffd80f2-42c6-4943-b507-36b4c23c569d", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0331 03:55:03.436894 3904 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-x6hq2 3e809162a393688cfe195be8481a870926a6c61f460e10d855bbf7e8d1f797d0 network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-x6hq2 3e809162a393688cfe195be8481a870926a6c61f460e10d855bbf7e8d1f797d0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:fa:18:05:ae:3b\",\"name\":\"3e809162a393688\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1311bdfd-8346-4658-a9d9-9e56a3262e6e\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:04.283664 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:04.283698 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 41.891µs I0331 03:55:04.283676 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-138.ec2.internal in network "default" I0331 03:55:04.283721 3904 node_tracker.go:220] Node ip-10-0-1-138.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0331 03:55:04.283732 3904 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall E0331 03:55:04.283768 3904 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-138.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0331 03:55:04.283787 3904 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-138.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-138.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-138.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-138.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-138.ec2.internal, error parsing annotation for node ip-10-0-1-138.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0331 03:55:04.295964 3904 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-138.ec2.internal annotation: unexpected end of JSON input I0331 03:55:04.295988 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0331 03:55:04.296039 3904 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-138.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-138.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-138.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-138.ec2.internal" E0331 03:55:04.296039 3904 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-138.ec2.internal annotation: unexpected end of JSON input I0331 03:55:04.310059 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:04.310093 3904 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall I0331 03:55:04.310120 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:04.310135 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 19.147µs I0331 03:55:04.312845 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0331 03:55:04.329578 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:04.348576 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:04.348596 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 22.87µs I0331 03:55:04.348626 3904 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 03:55:04.348673 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:04.541730 3904 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 03:55:04.541741 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:04.541757 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:04.541773 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 21.496µs I0331 03:55:04.587138 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:04.847953 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:06.284181 3904 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-8pdft] creating logical port openshift-insights_insights-runtime-extractor-8pdft for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:06.284321 3904 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-8pdft I0331 03:55:06.304914 3904 pod.go:62] [openshift-insights/insights-runtime-extractor-8pdft] pod update took 20.623965ms I0331 03:55:06.304939 3904 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-8pdft] addLogicalPort annotation time took 20.650113ms I0331 03:55:06.308768 3904 pods.go:271] [openshift-insights/insights-runtime-extractor-8pdft] addLogicalPort took 24.600371ms, libovsdb time 3.359597ms I0331 03:55:06.587358 3904 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8 c9efa4860603f25c349063898a0c90e6b9ee5b6dcceaa0c53232d658f1e8a5a8 network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8 c9efa4860603f25c349063898a0c90e6b9ee5b6dcceaa0c53232d658f1e8a5a8 network default NAD default] I0331 03:55:06.596467 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-4djw8, hostIfaceName: c9efa4860603f25, network: default, NAD default, SandboxID: "c9efa4860603f25c349063898a0c90e6b9ee5b6dcceaa0c53232d658f1e8a5a8", PCI device ID: , UID: "ab02287d-adce-4f1a-be33-ee1b2e76bb32", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0331 03:55:06.639994 3904 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8 c9efa4860603f25c349063898a0c90e6b9ee5b6dcceaa0c53232d658f1e8a5a8 network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-4djw8 c9efa4860603f25c349063898a0c90e6b9ee5b6dcceaa0c53232d658f1e8a5a8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:67:ca:7c:57:1a\",\"name\":\"c9efa4860603f25\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d960742b-ce1e-4689-b95e-b7bf19dc79a9\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:55:08.405329 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0331 03:55:08.619032 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0331 03:55:08.999125 3904 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-8557867658-pvb6f IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-8557867658-pvb6f: no pod IPs found W0331 03:55:09.044339 3904 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-68db74b777-79wr2 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-68db74b777-79wr2: no pod IPs found I0331 03:55:12.220434 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:12.220485 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:12.220498 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 18.797µs I0331 03:55:12.220530 3904 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall W0331 03:55:12.894639 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0331 03:55:13.381651 3904 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-75cc576b7d-nqvzq IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-75cc576b7d-nqvzq: no pod IPs found I0331 03:55:14.535365 3904 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-1-138.ec2.internal I0331 03:55:14.535376 3904 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-0-60.ec2.internal I0331 03:55:14.535395 3904 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-1-138.ec2.internal I0331 03:55:14.535401 3904 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-0-60.ec2.internal I0331 03:55:14.535477 3904 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-1-138.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-138.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-138.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-138.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-138.ec2.internal I0331 03:55:14.535477 3904 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-0-60.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-60.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-60.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-60.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-60.ec2.internal I0331 03:55:14.624638 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:14.949426 3904 obj_retry.go:464] Detected object openshift-image-registry/image-registry-68db74b777-7xc9h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:55:14.949474 3904 pods.go:176] Deleting pod: openshift-image-registry/image-registry-68db74b777-7xc9h I0331 03:55:14.950990 3904 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-68db74b777-7xc9h, ips: 10.128.0.5 I0331 03:55:14.951026 3904 obj_retry.go:464] Detected object openshift-image-registry/image-registry-68db74b777-7xc9h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:55:14.959884 3904 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-75cc576b7d-pd4zw] creating logical port openshift-image-registry_image-registry-75cc576b7d-pd4zw for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:14.960004 3904 kube.go:256] Updating pod openshift-image-registry/image-registry-75cc576b7d-pd4zw I0331 03:55:14.967381 3904 obj_retry.go:458] Detected object openshift-image-registry/image-registry-68db74b777-7xc9h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:55:14.967404 3904 obj_retry.go:458] Detected object openshift-image-registry/image-registry-68db74b777-7xc9h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:55:14.970123 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-68db74b777-7xc9h I0331 03:55:14.970149 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-68db74b777-7xc9h I0331 03:55:14.974495 3904 pod.go:62] [openshift-image-registry/image-registry-75cc576b7d-pd4zw] pod update took 14.514743ms I0331 03:55:14.974520 3904 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-75cc576b7d-pd4zw] addLogicalPort annotation time took 14.540644ms I0331 03:55:14.975811 3904 pods.go:271] [openshift-image-registry/image-registry-75cc576b7d-pd4zw] addLogicalPort took 15.936586ms, libovsdb time 854.073µs I0331 03:55:15.355419 3904 cni.go:403] [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default] I0331 03:55:15.363975 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-75cc576b7d-pd4zw, hostIfaceName: a5200eb70dac901, network: default, NAD default, SandboxID: "a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba", PCI device ID: , UID: "4ea980f3-b089-4729-9f32-b33317627847", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0331 03:55:15.407261 3904 cni.go:424] [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:f1:08:c7:ab:4e\",\"name\":\"a5200eb70dac901\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/99ade841-f096-401a-a867-66068bb34d56\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:16.594932 3904 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-9ktxq I0331 03:55:16.594947 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-64dxv I0331 03:55:16.594960 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-t4kxf I0331 03:55:16.594971 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-st94r I0331 03:55:16.594973 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-64dxv I0331 03:55:16.594977 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-t4kxf I0331 03:55:16.594978 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-st94r I0331 03:55:16.594980 3904 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nwlvp I0331 03:55:16.594995 3904 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nwlvp I0331 03:55:16.595021 3904 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-64dxv, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-64dxv: suppressed error logged: pod openshift-multus/network-metrics-daemon-64dxv: no pod IPs found I0331 03:55:16.595022 3904 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-st94r, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-st94r: suppressed error logged: pod openshift-network-diagnostics/network-check-target-st94r: no pod IPs found I0331 03:55:16.595028 3904 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-nwlvp, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nwlvp: suppressed error logged: pod openshift-multus/network-metrics-daemon-nwlvp: no pod IPs found I0331 03:55:16.595022 3904 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-t4kxf, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-t4kxf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-t4kxf: no pod IPs found I0331 03:55:16.594962 3904 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-9ktxq I0331 03:55:16.595050 3904 obj_retry.go:375] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-9ktxq, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-9ktxq: suppressed error logged: pod kube-system/global-pull-secret-syncer-9ktxq: no pod IPs found I0331 03:55:22.310387 3904 cni.go:403] [openshift-insights/insights-runtime-extractor-8pdft 245b1beb597c8ba267c837a7daa535d22d30fa99cd2a8494a33d13332546d5fd network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-8pdft 245b1beb597c8ba267c837a7daa535d22d30fa99cd2a8494a33d13332546d5fd network default NAD default] I0331 03:55:22.318720 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-8pdft, hostIfaceName: 245b1beb597c8ba, network: default, NAD default, SandboxID: "245b1beb597c8ba267c837a7daa535d22d30fa99cd2a8494a33d13332546d5fd", PCI device ID: , UID: "36a967ff-0bf9-4256-a7ed-920fe4abfa24", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0331 03:55:22.361783 3904 cni.go:424] [openshift-insights/insights-runtime-extractor-8pdft 245b1beb597c8ba267c837a7daa535d22d30fa99cd2a8494a33d13332546d5fd network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-8pdft 245b1beb597c8ba267c837a7daa535d22d30fa99cd2a8494a33d13332546d5fd network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:ea:c9:72:66:a8\",\"name\":\"245b1beb597c8ba\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/751798bf-d633-48a6-8ebf-b886c2c6f4a5\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:22.725923 3904 cni.go:403] [openshift-dns/dns-default-zcl7p 0fc5745696bdea5b293a896833560bcaeb192440dc0295157050e6ef8f8e9a9a network default NAD default] ADD starting CNI request [openshift-dns/dns-default-zcl7p 0fc5745696bdea5b293a896833560bcaeb192440dc0295157050e6ef8f8e9a9a network default NAD default] I0331 03:55:22.734582 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-zcl7p, hostIfaceName: 0fc5745696bdea5, network: default, NAD default, SandboxID: "0fc5745696bdea5b293a896833560bcaeb192440dc0295157050e6ef8f8e9a9a", PCI device ID: , UID: "b87b1c38-1dab-4a27-9b67-a9b33c445fa2", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0331 03:55:22.759046 3904 cni.go:403] [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default] I0331 03:55:22.769136 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6fb4878899-lt98g, hostIfaceName: 126de846e991175, network: default, NAD default, SandboxID: "126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545", PCI device ID: , UID: "516d98f5-ee02-479a-906c-77efa10bf324", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0331 03:55:22.774555 3904 cni.go:403] [openshift-ingress/router-default-ffc7ccb6f-vf8sr 4a9f7bd3f58ba66beeffa5102b63be86181df705d0418eedd73bb4ef372288f0 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-ffc7ccb6f-vf8sr 4a9f7bd3f58ba66beeffa5102b63be86181df705d0418eedd73bb4ef372288f0 network default NAD default] I0331 03:55:22.784714 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-ffc7ccb6f-vf8sr, hostIfaceName: 4a9f7bd3f58ba66, network: default, NAD default, SandboxID: "4a9f7bd3f58ba66beeffa5102b63be86181df705d0418eedd73bb4ef372288f0", PCI device ID: , UID: "5a69b790-8a44-47f2-a592-fb3bce10a3d7", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0331 03:55:22.787815 3904 cni.go:424] [openshift-dns/dns-default-zcl7p 0fc5745696bdea5b293a896833560bcaeb192440dc0295157050e6ef8f8e9a9a network default NAD default] ADD finished CNI request [openshift-dns/dns-default-zcl7p 0fc5745696bdea5b293a896833560bcaeb192440dc0295157050e6ef8f8e9a9a network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:e9:d6:eb:6d:b1\",\"name\":\"0fc5745696bdea5\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d22fd1e5-28b2-46b8-92a3-41f6e25677bf\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:22.792504 3904 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-8zl84 04cee5efc8c7c6c4678a2bfc9bb87251b7ea1f47c15c4f478b67cf54abb2a85c network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-8zl84 04cee5efc8c7c6c4678a2bfc9bb87251b7ea1f47c15c4f478b67cf54abb2a85c network default NAD default] I0331 03:55:22.804748 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-8zl84, hostIfaceName: 04cee5efc8c7c6c, network: default, NAD default, SandboxID: "04cee5efc8c7c6c4678a2bfc9bb87251b7ea1f47c15c4f478b67cf54abb2a85c", PCI device ID: , UID: "37cf7258-e6f4-4908-8877-3e70db4b740b", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0331 03:55:22.806993 3904 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs c5f6f9fdac071beb8c21b527bb2f76934cf7671908838210f3a5c6328a00ac43 network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs c5f6f9fdac071beb8c21b527bb2f76934cf7671908838210f3a5c6328a00ac43 network default NAD default] I0331 03:55:22.817270 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:22.823906 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-7fscs, hostIfaceName: c5f6f9fdac071be, network: default, NAD default, SandboxID: "c5f6f9fdac071beb8c21b527bb2f76934cf7671908838210f3a5c6328a00ac43", PCI device ID: , UID: "c07bb120-1f67-4980-bd6c-dbbbd7b266a4", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0331 03:55:22.827336 3904 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-wctw7 26a843fb9a42a631205f982a0e18837e89c159fa4445427a55008db895e4d6d7 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-wctw7 26a843fb9a42a631205f982a0e18837e89c159fa4445427a55008db895e4d6d7 network default NAD default] I0331 03:55:22.830365 3904 cni.go:424] [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:b5:b2:d6:66:a3\",\"name\":\"126de846e991175\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c1fd4abb-97cf-4040-bf6c-0369f8f17fed\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:22.840408 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-wctw7, hostIfaceName: 26a843fb9a42a63, network: default, NAD default, SandboxID: "26a843fb9a42a631205f982a0e18837e89c159fa4445427a55008db895e4d6d7", PCI device ID: , UID: "2aca72fc-bc4b-4342-8674-b452fdf8d540", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0331 03:55:22.852939 3904 cni.go:424] [openshift-ingress/router-default-ffc7ccb6f-vf8sr 4a9f7bd3f58ba66beeffa5102b63be86181df705d0418eedd73bb4ef372288f0 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-ffc7ccb6f-vf8sr 4a9f7bd3f58ba66beeffa5102b63be86181df705d0418eedd73bb4ef372288f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:3a:e6:21:57:95\",\"name\":\"4a9f7bd3f58ba66\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ad7d2a4-e21b-4ec7-ab00-b4ee11cbd792\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:22.875359 3904 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-8zl84 04cee5efc8c7c6c4678a2bfc9bb87251b7ea1f47c15c4f478b67cf54abb2a85c network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-8zl84 04cee5efc8c7c6c4678a2bfc9bb87251b7ea1f47c15c4f478b67cf54abb2a85c network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:3f:8b:fa:ed:9f\",\"name\":\"04cee5efc8c7c6c\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d298f5a-d162-4e46-a1b2-c670da012600\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:22.890993 3904 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs c5f6f9fdac071beb8c21b527bb2f76934cf7671908838210f3a5c6328a00ac43 network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-7fscs c5f6f9fdac071beb8c21b527bb2f76934cf7671908838210f3a5c6328a00ac43 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:f1:47:6d:65:b5\",\"name\":\"c5f6f9fdac071be\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c85388c-d810-4d37-a049-aa0232f57486\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:22.908497 3904 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-wctw7 26a843fb9a42a631205f982a0e18837e89c159fa4445427a55008db895e4d6d7 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-wctw7 26a843fb9a42a631205f982a0e18837e89c159fa4445427a55008db895e4d6d7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:ff:90:c3:c4:d4\",\"name\":\"26a843fb9a42a63\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a90749bd-0d09-48c9-961f-92757826d9cd\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:24.209830 3904 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 03:55:24.209863 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:24.209885 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 29.839µs I0331 03:55:24.209893 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:24.778789 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:26.657295 3904 cni.go:403] [openshift-multus/network-metrics-daemon-lk8gr 379b34317a78d499df9f0b91b368ee5802b3b89e221336d1b30e2d3406d6768a network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-lk8gr 379b34317a78d499df9f0b91b368ee5802b3b89e221336d1b30e2d3406d6768a network default NAD default] I0331 03:55:26.665887 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-lk8gr, hostIfaceName: 379b34317a78d49, network: default, NAD default, SandboxID: "379b34317a78d499df9f0b91b368ee5802b3b89e221336d1b30e2d3406d6768a", PCI device ID: , UID: "a4df493d-cc8f-44d4-8adf-e12f1679108e", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0331 03:55:26.718050 3904 cni.go:424] [openshift-multus/network-metrics-daemon-lk8gr 379b34317a78d499df9f0b91b368ee5802b3b89e221336d1b30e2d3406d6768a network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-lk8gr 379b34317a78d499df9f0b91b368ee5802b3b89e221336d1b30e2d3406d6768a network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:1f:99:f1:d6:41\",\"name\":\"379b34317a78d49\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/643765a7-0228-4650-890f-abf555bfeacb\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:55:32.727189 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:32.727234 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:32.727241 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:32.779907 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:32.809079 3904 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0331 03:55:32.822646 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:55:32.822837 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0331 03:55:32.828283 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:32.865287 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:32.865402 3904 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-7vzh2 for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:32.865566 3904 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2 I0331 03:55:32.896110 3904 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2] pod update took 30.586518ms I0331 03:55:32.896134 3904 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2] addLogicalPort annotation time took 30.617839ms I0331 03:55:32.897431 3904 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2] addLogicalPort took 32.049664ms, libovsdb time 902.891µs I0331 03:55:32.905290 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:32.939315 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:32.945379 3904 base_network_controller_pods.go:487] [default/openshift-console/console-5dfd8574fd-rf5pp] creating logical port openshift-console_console-5dfd8574fd-rf5pp for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:32.945497 3904 kube.go:256] Updating pod openshift-console/console-5dfd8574fd-rf5pp I0331 03:55:32.948032 3904 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-sbqpv] creating logical port openshift-console_downloads-56d45c498b-sbqpv for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:32.948168 3904 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-sbqpv I0331 03:55:32.984919 3904 pod.go:62] [openshift-console/console-5dfd8574fd-rf5pp] pod update took 39.450418ms I0331 03:55:32.984942 3904 base_network_controller_pods.go:937] [default/openshift-console/console-5dfd8574fd-rf5pp] addLogicalPort annotation time took 39.474459ms I0331 03:55:32.986222 3904 pods.go:271] [openshift-console/console-5dfd8574fd-rf5pp] addLogicalPort took 40.860829ms, libovsdb time 891.807µs I0331 03:55:32.989790 3904 pod.go:62] [openshift-console/downloads-56d45c498b-sbqpv] pod update took 41.668841ms I0331 03:55:32.989808 3904 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-sbqpv] addLogicalPort annotation time took 41.689149ms I0331 03:55:32.991083 3904 pods.go:271] [openshift-console/downloads-56d45c498b-sbqpv] addLogicalPort took 43.060758ms, libovsdb time 898.004µs I0331 03:55:33.220049 3904 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2 182490a2ee44adf76c15683cb04a3beca86e590a90eac0e22136253b98e8c21b network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2 182490a2ee44adf76c15683cb04a3beca86e590a90eac0e22136253b98e8c21b network default NAD default] I0331 03:55:33.228594 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-7vzh2, hostIfaceName: 182490a2ee44adf, network: default, NAD default, SandboxID: "182490a2ee44adf76c15683cb04a3beca86e590a90eac0e22136253b98e8c21b", PCI device ID: , UID: "1c4b7630-191f-4366-9166-aa2e1ffb1004", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0331 03:55:33.273016 3904 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2 182490a2ee44adf76c15683cb04a3beca86e590a90eac0e22136253b98e8c21b network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7vzh2 182490a2ee44adf76c15683cb04a3beca86e590a90eac0e22136253b98e8c21b network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:3d:14:32:3a:d7\",\"name\":\"182490a2ee44adf\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e97408b4-40bf-4525-96c6-07dd7d29f834\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:33.309176 3904 cni.go:403] [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default] ADD starting CNI request [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default] I0331 03:55:33.313059 3904 cni.go:403] [openshift-console/downloads-56d45c498b-sbqpv 3100e3b1d5fb6f020aa298cf074c98090b1869ce5146d220b64d25b07119f06d network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-sbqpv 3100e3b1d5fb6f020aa298cf074c98090b1869ce5146d220b64d25b07119f06d network default NAD default] I0331 03:55:33.319602 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5dfd8574fd-rf5pp, hostIfaceName: 5a5360e9e3cece2, network: default, NAD default, SandboxID: "5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df", PCI device ID: , UID: "ac3007f7-04c5-4234-b4c7-dad8acb8a4ce", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0331 03:55:33.322135 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-sbqpv, hostIfaceName: 3100e3b1d5fb6f0, network: default, NAD default, SandboxID: "3100e3b1d5fb6f020aa298cf074c98090b1869ce5146d220b64d25b07119f06d", PCI device ID: , UID: "4d38e606-70f8-4bc4-817f-d2448296953f", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0331 03:55:33.365152 3904 cni.go:424] [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default] ADD finished CNI request [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:99:0f:2f:40:73\",\"name\":\"5a5360e9e3cece2\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dd075ea5-bf63-4812-ac96-1acf3fb3f536\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:33.382264 3904 cni.go:424] [openshift-console/downloads-56d45c498b-sbqpv 3100e3b1d5fb6f020aa298cf074c98090b1869ce5146d220b64d25b07119f06d network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-sbqpv 3100e3b1d5fb6f020aa298cf074c98090b1869ce5146d220b64d25b07119f06d network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:8b:72:8d:d5:47\",\"name\":\"3100e3b1d5fb6f0\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/11397d26-20a3-4e5f-afbf-e7a4d17532f0\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:55:33.893541 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-138.ec2.internal in network "default" I0331 03:55:33.893593 3904 node_tracker.go:235] Node ip-10-0-1-138.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-138.ec2.internal" I0331 03:55:33.893604 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "ip-10-0-1-138.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:33.893628 3904 node_tracker.go:169] Node ip-10-0-1-138.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:33.893639 3904 services_controller.go:574] Full service sync requested for network=default W0331 03:55:33.893783 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:33.894119 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:33.894139 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:33.894146 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0331 03:55:33.968602 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:55:33.968638 3904 node_tracker.go:235] Node ip-10-0-0-60.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-60.ec2.internal" I0331 03:55:33.968658 3904 node_tracker.go:169] Node ip-10-0-0-60.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:33.968673 3904 services_controller.go:574] Full service sync requested for network=default I0331 03:55:33.968605 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "ip-10-0-0-60.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0331 03:55:33.968924 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:33.968945 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:33.968951 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:33.969244 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:33.970279 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "ip-10-0-1-138.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:34.008820 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-138.ec2.internal in network "default" I0331 03:55:34.008856 3904 default_network_controller.go:1008] Node "ip-10-0-1-138.ec2.internal" in remote zone "ip-10-0-1-138.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:34.008870 3904 node_tracker.go:169] Node ip-10-0-1-138.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.008884 3904 services_controller.go:574] Full service sync requested for network=default W0331 03:55:34.008964 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.008974 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.008978 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.009214 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:34.009564 3904 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-138.ec2.internal for the network default I0331 03:55:34.010822 3904 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-138.ec2.internal" on network "default" took: 1.278724ms I0331 03:55:34.103968 3904 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "ip-10-0-0-60.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:34.103998 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:55:34.104057 3904 node_tracker.go:169] Node ip-10-0-0-60.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.104070 3904 services_controller.go:574] Full service sync requested for network=default W0331 03:55:34.104295 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.104313 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.104319 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.104440 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:34.104732 3904 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-60.ec2.internal for the network default I0331 03:55:34.106069 3904 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-60.ec2.internal" on network "default" took: 1.355807ms I0331 03:55:34.440613 3904 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp for pod on switch ip-10-0-2-188.ec2.internal I0331 03:55:34.440732 3904 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp I0331 03:55:34.453720 3904 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp] pod update took 13.024217ms I0331 03:55:34.453803 3904 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp] addLogicalPort annotation time took 13.110828ms I0331 03:55:34.455426 3904 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp] addLogicalPort took 14.832227ms, libovsdb time 1.101675ms I0331 03:55:34.561134 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-138.ec2.internal in network "default" I0331 03:55:34.561167 3904 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 03:55:34.561224 3904 node_tracker.go:169] Node ip-10-0-1-138.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.561234 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:34.561243 3904 services_controller.go:574] Full service sync requested for network=default I0331 03:55:34.561248 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 17.938µs I0331 03:55:34.561251 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-138.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:55:34.561264 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-138.ec2.internal", IP: 10.0.1.138 to PMTUD blocking rules W0331 03:55:34.561481 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:34.561742 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.561759 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.561765 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0331 03:55:34.567937 3904 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-138.ec2.internal. Updating no re-route policies I0331 03:55:34.577289 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-138.ec2.internal", IP: 10.0.1.138 to PMTUD blocking rules I0331 03:55:34.577353 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-138.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:55:34.654452 3904 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall I0331 03:55:34.654543 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:34.654556 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 15.92µs I0331 03:55:34.654553 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-60.ec2.internal", IP: 10.0.0.60 to PMTUD blocking rules I0331 03:55:34.654563 3904 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:55:34.654571 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-60.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:55:34.654616 3904 node_tracker.go:169] Node ip-10-0-0-60.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.654635 3904 services_controller.go:574] Full service sync requested for network=default W0331 03:55:34.654918 3904 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.654934 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.654939 3904 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.655221 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:34.661331 3904 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-60.ec2.internal. Updating no re-route policies I0331 03:55:34.672332 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-60.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:55:34.672401 3904 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-60.ec2.internal", IP: 10.0.0.60 to PMTUD blocking rules I0331 03:55:34.817124 3904 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default] I0331 03:55:34.827685 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp, hostIfaceName: 42f701ab574fd7f, network: default, NAD default, SandboxID: "42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66", PCI device ID: , UID: "fd0648ad-a7f9-4f90-8ce8-fd80a80b1f10", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0331 03:55:34.878379 3904 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:e6:34:db:a4:1c\",\"name\":\"42f701ab574fd7f\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5544b5d9-fbf7-4dc7-a263-2fca86ec9a11\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:55:35.046265 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0331 03:55:35.071609 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0331 03:55:37.060865 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:55:38.015910 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:38.015930 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:38.015938 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0331 03:55:38.522170 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:38.522219 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 57.743µs I0331 03:55:38.598119 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:38.598174 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 59.507µs W0331 03:55:39.357964 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:40.395702 3904 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:44.101795 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:55:47.949586 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:49.114028 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0331 03:55:50.150588 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0331 03:55:54.141146 3904 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0331 03:55:56.466624 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:56.466654 3904 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:56.466663 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:57.507020 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:57.507055 3904 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:57.507062 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:59.450073 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0331 03:55:59.517146 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:55:59.517161 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:55:59.536859 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0331 03:56:00.188277 3904 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default] I0331 03:56:00.252184 3904 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp 42f701ab574fd7f59464ac15e19c401efbd02025ee8eebdebd8363139e8f4d66 network default NAD default], result "{}", err I0331 03:56:00.329318 3904 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.329353 3904 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp I0331 03:56:00.330748 3904 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp, ips: 10.128.0.29 I0331 03:56:00.330778 3904 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.330796 3904 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.330805 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0331 03:56:00.331028 3904 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.331055 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0331 03:56:00.493057 3904 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.493077 3904 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.493082 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.493086 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.501681 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:56:00.557085 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:00.557108 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:00.634163 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.634181 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.634186 3904 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.634190 3904 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:56:01.358832 3904 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:01.358853 3904 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:01.358871 3904 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:01.358881 3904 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:04.792412 3904 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0331 03:56:04.800715 3904 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0331 03:56:05.216118 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0331 03:56:05.216118 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0331 03:56:05.216209 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0331 03:56:06.215561 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0331 03:56:10.249390 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0331 03:56:10.249444 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0331 03:56:10.249456 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0331 03:56:10.531162 3904 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0331 03:56:11.261037 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0331 03:56:16.267246 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0331 03:56:16.267263 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0331 03:56:16.267304 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0331 03:56:17.275324 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0331 03:56:20.944904 3904 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:20.945445 3904 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:21.573382 3904 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:21.573402 3904 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:21.578090 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dfd8574fd-6hg6d I0331 03:56:21.578113 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dfd8574fd-6hg6d I0331 03:56:25.639987 3904 cni.go:403] [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default] I0331 03:56:25.712596 3904 cni.go:424] [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-75cc576b7d-pd4zw a5200eb70dac901b40b0ab085ea0092b0155868b4b0499cedcd50d2a8d7420ba network default NAD default], result "{}", err I0331 03:56:25.789807 3904 obj_retry.go:464] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:25.789836 3904 pods.go:176] Deleting pod: openshift-image-registry/image-registry-75cc576b7d-pd4zw I0331 03:56:25.791163 3904 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-75cc576b7d-pd4zw, ips: 10.128.0.25 I0331 03:56:25.791210 3904 obj_retry.go:464] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:26.315160 3904 obj_retry.go:458] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:26.315210 3904 obj_retry.go:458] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:26.320658 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-75cc576b7d-pd4zw I0331 03:56:26.320695 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-75cc576b7d-pd4zw I0331 03:56:31.750239 3904 cni.go:403] [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default] DEL starting CNI request [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default] I0331 03:56:31.826277 3904 cni.go:424] [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default] DEL finished CNI request [openshift-console/console-5dfd8574fd-rf5pp 5a5360e9e3cece294e1e470f3b23e608b6397e4b9b9056baf376ad29bcac45df network default NAD default], result "{}", err I0331 03:56:31.896117 3904 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:31.896151 3904 pods.go:176] Deleting pod: openshift-console/console-5dfd8574fd-rf5pp I0331 03:56:31.897422 3904 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5dfd8574fd-rf5pp, ips: 10.128.0.27 I0331 03:56:31.897451 3904 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:32.338512 3904 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:32.338541 3904 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:32.345643 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dfd8574fd-rf5pp I0331 03:56:32.345662 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dfd8574fd-rf5pp I0331 03:56:40.220412 3904 cni.go:403] [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default] DEL starting CNI request [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default] I0331 03:56:40.292545 3904 cni.go:424] [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default] DEL finished CNI request [openshift-ingress/router-default-6fb4878899-lt98g 126de846e9911757105961120f085d246ac6f86f851e9f4b57ad5b1687f9f545 network default NAD default], result "{}", err I0331 03:56:40.355456 3904 obj_retry.go:464] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:40.355495 3904 pods.go:176] Deleting pod: openshift-ingress/router-default-6fb4878899-lt98g I0331 03:56:40.356992 3904 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-6fb4878899-lt98g, ips: 10.128.0.10 I0331 03:56:40.357033 3904 obj_retry.go:464] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:40.678809 3904 obj_retry.go:458] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:40.678833 3904 obj_retry.go:458] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:40.681890 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-6fb4878899-lt98g I0331 03:56:40.681911 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-6fb4878899-lt98g I0331 03:57:18.065674 3904 base_network_controller_pods.go:487] [default/openshift-console/console-796968dcf4-zd56g] creating logical port openshift-console_console-796968dcf4-zd56g for pod on switch ip-10-0-2-188.ec2.internal I0331 03:57:18.065781 3904 kube.go:256] Updating pod openshift-console/console-796968dcf4-zd56g I0331 03:57:18.083966 3904 pod.go:62] [openshift-console/console-796968dcf4-zd56g] pod update took 18.222737ms I0331 03:57:18.083996 3904 base_network_controller_pods.go:937] [default/openshift-console/console-796968dcf4-zd56g] addLogicalPort annotation time took 18.257256ms I0331 03:57:18.085621 3904 pods.go:271] [openshift-console/console-796968dcf4-zd56g] addLogicalPort took 19.96892ms, libovsdb time 1.133063ms I0331 03:57:18.429997 3904 cni.go:403] [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default] ADD starting CNI request [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default] I0331 03:57:18.446417 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-796968dcf4-zd56g, hostIfaceName: f86de9fdcb34430, network: default, NAD default, SandboxID: "f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0", PCI device ID: , UID: "ff81e32a-41f8-406e-899e-297222936ac9", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0331 03:57:18.493364 3904 cni.go:424] [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default] ADD finished CNI request [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:48:71:23:36:f5\",\"name\":\"f86de9fdcb34430\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29c61d23-83f5-4b60-a095-cc30739a7563\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:57:28.599628 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:57:28.738695 3904 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-796968dcf4-xj7wl IP address from the namespace address-set, err: pod openshift-console/console-796968dcf4-xj7wl: no pod IPs found I0331 03:57:43.376215 3904 obj_retry.go:464] Detected object openshift-console/console-6877b44645-tchg7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:43.376792 3904 obj_retry.go:464] Detected object openshift-console/console-6877b44645-tchg7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:43.888339 3904 obj_retry.go:458] Detected object openshift-console/console-6877b44645-tchg7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:43.888360 3904 obj_retry.go:458] Detected object openshift-console/console-6877b44645-tchg7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:43.895386 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6877b44645-tchg7 I0331 03:57:43.895402 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6877b44645-tchg7 I0331 03:57:53.916097 3904 obj_retry.go:464] Detected object openshift-console/console-6877b44645-xxckv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:53.916636 3904 obj_retry.go:464] Detected object openshift-console/console-6877b44645-xxckv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:54.247572 3904 obj_retry.go:458] Detected object openshift-console/console-6877b44645-xxckv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:54.247592 3904 obj_retry.go:458] Detected object openshift-console/console-6877b44645-xxckv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:54.254070 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6877b44645-xxckv I0331 03:57:54.254091 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6877b44645-xxckv I0331 03:58:00.963825 3904 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0331 03:58:19.149412 3904 cni.go:403] [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default] DEL starting CNI request [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default] I0331 03:58:19.225919 3904 cni.go:424] [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default] DEL finished CNI request [openshift-console/console-796968dcf4-zd56g f86de9fdcb34430f0a61e0f57320cd2fdf112594dffae1191f6c70d2e61530f0 network default NAD default], result "{}", err I0331 03:58:19.271958 3904 obj_retry.go:464] Detected object openshift-console/console-796968dcf4-zd56g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:58:19.271983 3904 pods.go:176] Deleting pod: openshift-console/console-796968dcf4-zd56g I0331 03:58:19.273291 3904 pods.go:236] Attempting to release IPs for pod: openshift-console/console-796968dcf4-zd56g, ips: 10.128.0.30 I0331 03:58:19.273320 3904 obj_retry.go:464] Detected object openshift-console/console-796968dcf4-zd56g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:58:19.782535 3904 obj_retry.go:458] Detected object openshift-console/console-796968dcf4-zd56g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:58:19.782563 3904 obj_retry.go:458] Detected object openshift-console/console-796968dcf4-zd56g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:58:19.790107 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-796968dcf4-zd56g I0331 03:58:19.790131 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-796968dcf4-zd56g I0331 03:59:52.196776 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0331 03:59:55.072526 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0331 04:00:03.073402 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0331 04:00:04.202281 3904 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.202318 3904 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.620602 3904 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.621174 3904 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:05.210155 3904 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.210181 3904 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.506897 3904 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.506919 3904 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:11.191123 3904 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0331 04:00:21.072593 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=120 I0331 04:00:45.514392 3904 namespace.go:98] [tsf] adding namespace I0331 04:00:45.515267 3904 namespace.go:102] [tsf] adding namespace took 849.882µs I0331 04:00:45.528836 3904 namespace.go:142] [tsf] updating namespace I0331 04:00:45.541348 3904 namespace.go:142] [tsf] updating namespace I0331 04:00:49.698738 3904 namespace.go:98] [tsf-tpa] adding namespace I0331 04:00:49.698747 3904 namespace.go:98] [rhtpa-operator] adding namespace I0331 04:00:49.699252 3904 namespace.go:98] [tsf-tas] adding namespace I0331 04:00:49.699328 3904 namespace.go:98] [konflux-ui] adding namespace I0331 04:00:49.699727 3904 namespace.go:102] [tsf-tpa] adding namespace took 968.677µs I0331 04:00:49.700402 3904 namespace.go:102] [rhtpa-operator] adding namespace took 1.636415ms I0331 04:00:49.700938 3904 namespace.go:102] [tsf-tas] adding namespace took 1.671753ms I0331 04:00:49.701475 3904 namespace.go:102] [konflux-ui] adding namespace took 2.136326ms I0331 04:00:49.702403 3904 namespace.go:98] [rhbk-operator] adding namespace I0331 04:00:49.702525 3904 namespace.go:98] [cert-manager-operator] adding namespace I0331 04:00:49.702589 3904 namespace.go:98] [konflux-operator] adding namespace I0331 04:00:49.702657 3904 namespace.go:98] [tsf-keycloak] adding namespace I0331 04:00:49.703019 3904 namespace.go:102] [rhbk-operator] adding namespace took 604.562µs I0331 04:00:49.703498 3904 namespace.go:102] [cert-manager-operator] adding namespace took 956.202µs I0331 04:00:49.704018 3904 namespace.go:102] [konflux-operator] adding namespace took 1.411282ms I0331 04:00:49.704637 3904 namespace.go:102] [tsf-keycloak] adding namespace took 1.963422ms I0331 04:00:49.714466 3904 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:49.728290 3904 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:49.734220 3904 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:00:49.735376 3904 namespace.go:142] [rhbk-operator] updating namespace I0331 04:00:49.735487 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:00:49.740814 3904 namespace.go:142] [tsf-tas] updating namespace I0331 04:00:49.750840 3904 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:00:49.754410 3904 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:00:49.754464 3904 namespace.go:142] [konflux-operator] updating namespace I0331 04:00:49.754565 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:49.754693 3904 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:49.788736 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:00:49.798927 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:00:49.806501 3904 namespace.go:142] [rhbk-operator] updating namespace I0331 04:00:49.818518 3904 namespace.go:142] [rhbk-operator] updating namespace I0331 04:00:49.830284 3904 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:00:49.841247 3904 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:00:49.848813 3904 namespace.go:142] [konflux-operator] updating namespace I0331 04:00:49.858211 3904 namespace.go:142] [konflux-operator] updating namespace I0331 04:00:49.867691 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:49.879332 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:49.884032 3904 namespace.go:142] [tsf-tas] updating namespace I0331 04:00:49.894819 3904 namespace.go:142] [tsf-tas] updating namespace I0331 04:00:49.904109 3904 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:00:50.816891 3904 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:50.828809 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:50.967525 3904 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:50.967643 3904 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0331 04:00:50.984898 3904 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 17.279269ms I0331 04:00:50.984918 3904 base_network_controller_pods.go:937] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 17.305509ms I0331 04:00:50.986102 3904 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 18.593511ms, libovsdb time 821.69µs I0331 04:00:51.332929 3904 cni.go:403] [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default] ADD starting CNI request [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default] I0331 04:00:51.341586 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 9b2f430d5eb956c, network: default, NAD default, SandboxID: "9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564", PCI device ID: , UID: "0ebdaa86-3ac0-4784-8d59-44679955b4a3", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0331 04:00:51.386678 3904 cni.go:424] [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default] ADD finished CNI request [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:2b:f2:a6:25:f7\",\"name\":\"9b2f430d5eb956c\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da708702-d936-4084-b83b-64dbca7259a7\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:52.386587 3904 base_network_controller_pods.go:487] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x] creating logical port openshift-marketplace_867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:52.386754 3904 kube.go:256] Updating pod openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x I0331 04:00:52.399228 3904 pod.go:62] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x] pod update took 12.51541ms I0331 04:00:52.399247 3904 base_network_controller_pods.go:937] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x] addLogicalPort annotation time took 12.54158ms I0331 04:00:52.400650 3904 pods.go:271] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x] addLogicalPort took 14.078035ms, libovsdb time 1.064066ms I0331 04:00:52.490499 3904 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:52.490624 3904 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn I0331 04:00:52.502667 3904 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn] pod update took 12.067353ms I0331 04:00:52.502691 3904 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn] addLogicalPort annotation time took 12.09546ms I0331 04:00:52.504088 3904 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn] addLogicalPort took 13.609233ms, libovsdb time 930.838µs I0331 04:00:53.134928 3904 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default] ADD starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default] I0331 04:00:53.135259 3904 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default] I0331 04:00:53.146122 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x, hostIfaceName: 06ae6c2132b746c, network: default, NAD default, SandboxID: "06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895", PCI device ID: , UID: "4514d765-2a8a-4f5f-97b4-5096fe9d8be9", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0331 04:00:53.146826 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn, hostIfaceName: 71b5fdc4774732f, network: default, NAD default, SandboxID: "71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a", PCI device ID: , UID: "94cdaa07-7f6b-4a6d-90da-bd6e93a95bed", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0331 04:00:53.193261 3904 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default] ADD finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:e4:09:e5:89:a6\",\"name\":\"06ae6c2132b746c\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/81417135-f740-46c7-b7ab-8fc68ecda0ab\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:53.210991 3904 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:2e:81:d7:2d:f3\",\"name\":\"71b5fdc4774732f\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b74a8f0e-e922-4cc5-8608-be272211d8ed\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:55.501152 3904 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:55.501278 3904 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb I0331 04:00:55.515696 3904 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb] pod update took 14.449228ms I0331 04:00:55.515725 3904 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb] addLogicalPort annotation time took 14.482234ms I0331 04:00:55.517512 3904 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb] addLogicalPort took 16.377286ms, libovsdb time 1.353377ms I0331 04:00:55.704392 3904 base_network_controller_pods.go:487] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk] creating logical port openshift-marketplace_ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:55.704551 3904 kube.go:256] Updating pod openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk I0331 04:00:55.719638 3904 pod.go:62] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk] pod update took 15.119772ms I0331 04:00:55.719671 3904 base_network_controller_pods.go:937] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk] addLogicalPort annotation time took 15.15657ms I0331 04:00:55.721262 3904 pods.go:271] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk] addLogicalPort took 16.890496ms, libovsdb time 1.022577ms I0331 04:00:56.571917 3904 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default] ADD starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default] I0331 04:00:56.572095 3904 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default] I0331 04:00:56.585113 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb, hostIfaceName: 367f30953f913e0, network: default, NAD default, SandboxID: "367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3", PCI device ID: , UID: "4f015944-cbc3-452f-a349-8cd8eb41ad6f", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0331 04:00:56.586136 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk, hostIfaceName: 8d5b7bd687bbbd4, network: default, NAD default, SandboxID: "8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216", PCI device ID: , UID: "5e4355e8-7c07-4017-8fd6-052c77991cac", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0331 04:00:56.641596 3904 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default] ADD finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:40:a8:12:79:c9\",\"name\":\"8d5b7bd687bbbd4\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/52c8e918-b0ce-446b-8629-64106813108d\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:56.666861 3904 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:17:0e:7a:32:ab\",\"name\":\"367f30953f913e0\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3666b191-1f41-43e9-84e4-761d66701aad\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:58.444391 3904 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default] DEL starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default] I0331 04:00:58.462586 3904 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:58.462697 3904 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz I0331 04:00:58.475520 3904 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz] pod update took 12.860492ms I0331 04:00:58.475546 3904 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz] addLogicalPort annotation time took 12.88957ms I0331 04:00:58.476847 3904 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz] addLogicalPort took 14.27739ms, libovsdb time 910.837µs I0331 04:00:58.518090 3904 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default] DEL finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x 06ae6c2132b746ca2ac59fbf422bf616a069889b692fea5da0be47469e79f895 network default NAD default], result "{}", err I0331 04:00:58.563016 3904 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:00:58.563133 3904 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 I0331 04:00:58.576331 3904 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422] pod update took 13.237252ms I0331 04:00:58.576418 3904 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422] addLogicalPort annotation time took 13.334216ms I0331 04:00:58.577943 3904 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422] addLogicalPort took 14.946159ms, libovsdb time 1.056815ms I0331 04:00:59.835442 3904 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default] I0331 04:00:59.835632 3904 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default] I0331 04:00:59.856588 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz, hostIfaceName: a106e7a3bf59e18, network: default, NAD default, SandboxID: "a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3", PCI device ID: , UID: "bc74aa5f-a8b9-4cb3-979d-8582731f55ad", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0331 04:00:59.869212 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422, hostIfaceName: b8b716eb9da8b3d, network: default, NAD default, SandboxID: "b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6", PCI device ID: , UID: "1d1f9c03-a40f-4b75-a337-a1ad44183941", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0331 04:00:59.870375 3904 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.870424 3904 pods.go:176] Deleting pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x I0331 04:00:59.871916 3904 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x, ips: 10.128.0.32 I0331 04:00:59.871949 3904 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.871979 3904 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.871993 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:00:59.872243 3904 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.872259 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:00:59.909697 3904 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:67:e4:b8:89:e6\",\"name\":\"a106e7a3bf59e18\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dba7c099-05ff-45ef-a05f-a4c047d62b5e\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:59.940292 3904 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:b5:78:4a:36:f3\",\"name\":\"b8b716eb9da8b3d\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b33f73e-2568-4aa7-891f-4a5e08d72d0d\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:01:00.895070 3904 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:00.895115 3904 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:00.895144 3904 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:00.895157 3904 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:01.481576 3904 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default] I0331 04:01:01.548341 3904 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn 71b5fdc4774732fb37e5247b912560d208439910baba0df1e08118eb8daa2e3a network default NAD default], result "{}", err I0331 04:01:01.615918 3904 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.615954 3904 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn I0331 04:01:01.617505 3904 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn, ips: 10.128.0.33 I0331 04:01:01.617535 3904 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.617559 3904 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.617568 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:01.617765 3904 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.617780 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:02.493519 3904 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default] DEL starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default] I0331 04:01:02.563179 3904 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default] DEL finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk 8d5b7bd687bbbd44277ee6254d2e5f34bf1ae5d7c01d8fceb278f8fac443b216 network default NAD default], result "{}", err I0331 04:01:02.601665 3904 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.601708 3904 pods.go:176] Deleting pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk I0331 04:01:02.603412 3904 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk, ips: 10.128.0.35 I0331 04:01:02.603445 3904 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.603468 3904 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.603479 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:02.603751 3904 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.603808 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:02.643948 3904 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:02.643970 3904 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:02.643987 3904 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:02.643995 3904 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.636014 3904 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.636059 3904 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.636076 3904 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.636084 3904 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:04.155281 3904 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0331 04:01:04.512030 3904 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default] I0331 04:01:04.512271 3904 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default] I0331 04:01:04.512303 3904 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default] I0331 04:01:04.609422 3904 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb 367f30953f913e052ebb5706b57e184071512be150b314e7d835fdd88ef37aa3 network default NAD default], result "{}", err I0331 04:01:04.632897 3904 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 b8b716eb9da8b3d89eb5143755970b50c1f7ff579fc1faa0d4534e0adae671a6 network default NAD default], result "{}", err I0331 04:01:04.633378 3904 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz a106e7a3bf59e18228f3aea82e874440485b6aa77e3f8e0e5473fc54995baeb3 network default NAD default], result "{}", err I0331 04:01:04.673121 3904 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.673160 3904 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb I0331 04:01:04.674858 3904 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb, ips: 10.128.0.34 I0331 04:01:04.674896 3904 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.674919 3904 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.674929 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:04.675169 3904 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.675184 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:04.705190 3904 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.705290 3904 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz I0331 04:01:04.706472 3904 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz, ips: 10.128.0.36 I0331 04:01:04.706496 3904 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.706516 3904 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.706525 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:04.706683 3904 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.706695 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:04.735025 3904 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.735054 3904 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 I0331 04:01:04.736153 3904 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422, ips: 10.128.0.37 I0331 04:01:04.736173 3904 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.736186 3904 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.736212 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:04.736381 3904 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.736394 3904 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:05.700081 3904 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.700122 3904 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.700138 3904 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.700151 3904 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.731000 3904 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.731022 3904 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.731036 3904 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.731045 3904 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760436 3904 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760484 3904 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760496 3904 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760506 3904 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:08.195784 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 W0331 04:01:12.682643 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:15.573613 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:15.584065 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0331 04:01:15.797237 3904 namespace.go:142] [openshift-operators] updating namespace I0331 04:01:15.808823 3904 namespace.go:142] [openshift-operators] updating namespace W0331 04:01:16.175996 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:16.187008 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:16.196842 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:16.204899 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:19.839044 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:01:24.789050 3904 namespace.go:142] [openshift-operators] updating namespace W0331 04:01:28.334554 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:28.348854 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:28.467993 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:28.468070 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:01:34.103144 3904 namespace.go:98] [trusted-artifact-signer] adding namespace I0331 04:01:34.104051 3904 namespace.go:102] [trusted-artifact-signer] adding namespace took 882.652µs I0331 04:01:34.134724 3904 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:34.142897 3904 namespace.go:142] [trusted-artifact-signer] updating namespace W0331 04:01:34.791474 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0331 04:01:39.243862 3904 namespace.go:98] [cert-manager] adding namespace I0331 04:01:39.244767 3904 namespace.go:102] [cert-manager] adding namespace took 880.928µs I0331 04:01:39.276900 3904 namespace.go:142] [cert-manager] updating namespace I0331 04:01:39.291785 3904 namespace.go:142] [cert-manager] updating namespace I0331 04:01:40.661012 3904 namespace.go:98] [openshift-pipelines] adding namespace I0331 04:01:40.661857 3904 namespace.go:102] [openshift-pipelines] adding namespace took 821.98µs I0331 04:01:40.719022 3904 namespace.go:142] [openshift-pipelines] updating namespace I0331 04:01:41.587895 3904 namespace.go:142] [cert-manager] updating namespace I0331 04:01:41.794919 3904 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:42.203638 3904 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:42.402927 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:42.608771 3904 namespace.go:142] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] updating namespace I0331 04:01:43.054915 3904 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:43.279079 3904 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:43.477857 3904 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:43.687823 3904 namespace.go:142] [tsf] updating namespace I0331 04:01:43.875874 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:44.083863 3904 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:44.286720 3904 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:44.417687 3904 namespace.go:142] [cert-manager] updating namespace I0331 04:01:44.426413 3904 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:44.439277 3904 namespace.go:142] [default] updating namespace I0331 04:01:44.449991 3904 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:44.460654 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:44.474825 3904 namespace.go:142] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] updating namespace I0331 04:01:44.480871 3904 namespace.go:142] [openshift] updating namespace I0331 04:01:44.498526 3904 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:44.512719 3904 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:44.521238 3904 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:44.531934 3904 namespace.go:142] [tsf] updating namespace I0331 04:01:44.542475 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:44.553536 3904 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:44.564786 3904 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:44.660312 3904 namespace.go:142] [cert-manager] updating namespace I0331 04:01:44.764745 3904 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:44.859370 3904 namespace.go:142] [default] updating namespace I0331 04:01:44.961850 3904 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:45.058967 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:45.159574 3904 namespace.go:142] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] updating namespace I0331 04:01:45.332810 3904 namespace.go:142] [openshift] updating namespace I0331 04:01:45.451866 3904 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:45.546885 3904 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:45.668812 3904 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:45.824711 3904 namespace.go:142] [tsf] updating namespace I0331 04:01:45.930562 3904 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:46.052981 3904 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:46.151982 3904 namespace.go:142] [tsf-tpa] updating namespace W0331 04:01:46.968353 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:01:46.968369 3904 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0331 04:01:47.085895 3904 namespace.go:142] [openshift-pipelines] updating namespace W0331 04:01:57.909573 3904 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:57.909593 3904 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:57.909601 3904 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:57.909605 3904 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:59.080769 3904 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:01:59.080790 3904 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:01:59.080795 3904 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:01:59.107972 3904 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:01:59.107985 3904 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:01:59.107989 3904 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0331 04:02:00.131125 3904 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-jnr6k] creating logical port cert-manager_cert-manager-59ccb4bb6b-jnr6k for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:00.131234 3904 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-jnr6k I0331 04:02:00.146122 3904 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-jnr6k] pod update took 14.931113ms I0331 04:02:00.146149 3904 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-jnr6k] addLogicalPort annotation time took 14.961987ms I0331 04:02:00.147407 3904 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-jnr6k] addLogicalPort took 16.297454ms, libovsdb time 893.939µs I0331 04:02:00.490322 3904 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-jnr6k 1adc5089154b81cdbd2c8df890f6d71cfa0b4d9c4f220525edfbf011018a92a6 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-jnr6k 1adc5089154b81cdbd2c8df890f6d71cfa0b4d9c4f220525edfbf011018a92a6 network default NAD default] I0331 04:02:00.498800 3904 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-jnr6k, hostIfaceName: 1adc5089154b81c, network: default, NAD default, SandboxID: "1adc5089154b81cdbd2c8df890f6d71cfa0b4d9c4f220525edfbf011018a92a6", PCI device ID: , UID: "0d2297aa-96a3-4040-98a6-d324b7d24916", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0331 04:02:00.543629 3904 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-jnr6k 1adc5089154b81cdbd2c8df890f6d71cfa0b4d9c4f220525edfbf011018a92a6 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-jnr6k 1adc5089154b81cdbd2c8df890f6d71cfa0b4d9c4f220525edfbf011018a92a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:f9:c5:6a:c6:c7\",\"name\":\"1adc5089154b81c\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea853a53-277c-4111-b571-6fd3115209dc\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:03.834476 3904 cni.go:403] [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default] DEL starting CNI request [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default] I0331 04:02:03.904717 3904 cni.go:424] [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default] DEL finished CNI request [tsf/test-tsf-subscriptions 9b2f430d5eb956cfd4f124d9a6473d8f8c2900c6a30ad5fc06b01dd36cb5d564 network default NAD default], result "{}", err I0331 04:02:03.958079 3904 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:03.958116 3904 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0331 04:02:03.959427 3904 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.31 I0331 04:02:03.959457 3904 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:03.984073 3904 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:03.984095 3904 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:03.991471 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0331 04:02:03.991491 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0331 04:02:05.150932 3904 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:05.151031 3904 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0331 04:02:05.166225 3904 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 15.219216ms I0331 04:02:05.166245 3904 base_network_controller_pods.go:937] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 15.242957ms I0331 04:02:05.167717 3904 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 16.798365ms, libovsdb time 1.049242ms I0331 04:02:05.516270 3904 cni.go:403] [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default] ADD starting CNI request [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default] I0331 04:02:05.524750 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 673d5cb891d439a, network: default, NAD default, SandboxID: "673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c", PCI device ID: , UID: "67ae044e-c05e-4f0f-994a-2a377ce3d4fd", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0331 04:02:05.570550 3904 cni.go:424] [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default] ADD finished CNI request [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:7a:f3:2c:43:70\",\"name\":\"673d5cb891d439a\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5abd0d87-8a04-492d-a7c5-7271c093ec00\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:08.035517 3904 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-n2h6s for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:08.035630 3904 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s I0331 04:02:08.042799 3904 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-wct4k for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:08.042981 3904 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k I0331 04:02:08.047658 3904 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s] pod update took 12.06859ms I0331 04:02:08.047767 3904 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s] addLogicalPort annotation time took 12.180272ms I0331 04:02:08.049524 3904 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s] addLogicalPort took 14.02471ms, libovsdb time 1.09874ms I0331 04:02:08.055290 3904 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k] pod update took 12.352422ms I0331 04:02:08.055341 3904 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k] addLogicalPort annotation time took 12.407032ms I0331 04:02:08.056859 3904 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k] addLogicalPort took 14.080314ms, libovsdb time 1.013065ms I0331 04:02:12.547551 3904 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 04:02:12.547625 3904 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 04:02:12.547650 3904 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 32.207µs I0331 04:02:13.198054 3904 cni.go:403] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s 60cd0dda7d2a8a9c3809a4d1607e0867072f316c228a8694e7f56f23631d3c67 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s 60cd0dda7d2a8a9c3809a4d1607e0867072f316c228a8694e7f56f23631d3c67 network default NAD default] I0331 04:02:13.207297 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-n2h6s, hostIfaceName: 60cd0dda7d2a8a9, network: default, NAD default, SandboxID: "60cd0dda7d2a8a9c3809a4d1607e0867072f316c228a8694e7f56f23631d3c67", PCI device ID: , UID: "96586f5f-b980-4f0e-b1dd-8d83868b9871", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0331 04:02:13.253081 3904 cni.go:424] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s 60cd0dda7d2a8a9c3809a4d1607e0867072f316c228a8694e7f56f23631d3c67 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n2h6s 60cd0dda7d2a8a9c3809a4d1607e0867072f316c228a8694e7f56f23631d3c67 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:77:e1:30:56:15\",\"name\":\"60cd0dda7d2a8a9\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2de874d3-facf-4477-8119-b4773f246f1c\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:13.510139 3904 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k 78fb2c29718afb81e48b71f7a10584cefb0fb7b3e2e5d30e7a3409ff07a3f949 network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k 78fb2c29718afb81e48b71f7a10584cefb0fb7b3e2e5d30e7a3409ff07a3f949 network default NAD default] I0331 04:02:13.519141 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-wct4k, hostIfaceName: 78fb2c29718afb8, network: default, NAD default, SandboxID: "78fb2c29718afb81e48b71f7a10584cefb0fb7b3e2e5d30e7a3409ff07a3f949", PCI device ID: , UID: "2eb720a2-00a4-4301-b643-4e5e362f0085", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0331 04:02:13.564761 3904 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k 78fb2c29718afb81e48b71f7a10584cefb0fb7b3e2e5d30e7a3409ff07a3f949 network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-wct4k 78fb2c29718afb81e48b71f7a10584cefb0fb7b3e2e5d30e7a3409ff07a3f949 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:06:78:ac:08:90\",\"name\":\"78fb2c29718afb8\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/75e0cc26-e70f-4de3-8d1e-822d641d4564\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:02:13.889102 3904 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0331 04:02:13.912531 3904 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0331 04:02:20.072265 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=101 I0331 04:02:22.596753 3904 namespace.go:142] [openshift-pipelines] updating namespace W0331 04:02:23.035366 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0331 04:02:25.166553 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 W0331 04:02:34.901131 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0331 04:02:34.928479 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0331 04:02:35.160282 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0331 04:02:35.897740 3904 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-7sv9b for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:35.897829 3904 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b I0331 04:02:35.915012 3904 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b] pod update took 17.20441ms I0331 04:02:35.915036 3904 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b] addLogicalPort annotation time took 17.23525ms I0331 04:02:35.916302 3904 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b] addLogicalPort took 18.579752ms, libovsdb time 882.874µs I0331 04:02:36.259612 3904 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b 5474e175037571b26925840971d216ce4f1c5b2ed5748e217934bd8e3fee16d7 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b 5474e175037571b26925840971d216ce4f1c5b2ed5748e217934bd8e3fee16d7 network default NAD default] I0331 04:02:36.268279 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-7sv9b, hostIfaceName: 5474e175037571b, network: default, NAD default, SandboxID: "5474e175037571b26925840971d216ce4f1c5b2ed5748e217934bd8e3fee16d7", PCI device ID: , UID: "3390250e-a862-4581-bd58-e1c72d6c58a8", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0331 04:02:36.313367 3904 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b 5474e175037571b26925840971d216ce4f1c5b2ed5748e217934bd8e3fee16d7 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-7sv9b 5474e175037571b26925840971d216ce4f1c5b2ed5748e217934bd8e3fee16d7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:84:7f:5f:75:17\",\"name\":\"5474e175037571b\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0e36afeb-5b54-4541-b628-8332e1fda413\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:37.016856 3904 cni.go:403] [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default] DEL starting CNI request [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default] I0331 04:02:37.091851 3904 cni.go:424] [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default] DEL finished CNI request [tsf/test-tsf-infrastructure 673d5cb891d439ac40d55b17e4004f8da0d0385bc0b55f68fc92c6479757116c network default NAD default], result "{}", err I0331 04:02:37.144304 3904 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:37.144339 3904 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0331 04:02:37.145669 3904 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.39 I0331 04:02:37.145696 3904 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:37.164374 3904 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:37.164399 3904 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:37.170373 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0331 04:02:37.170420 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0331 04:02:38.336755 3904 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:38.336880 3904 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0331 04:02:38.355069 3904 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 18.226384ms I0331 04:02:38.355097 3904 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 18.26161ms I0331 04:02:38.356529 3904 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 19.793437ms, libovsdb time 989.631µs I0331 04:02:38.411104 3904 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-188.ec2.internal I0331 04:02:38.411272 3904 kube.go:256] Updating pod tsf/test-tsf-iam I0331 04:02:38.423802 3904 pod.go:62] [tsf/test-tsf-iam] pod update took 12.566631ms I0331 04:02:38.423827 3904 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 12.595761ms I0331 04:02:38.425237 3904 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 14.154086ms, libovsdb time 947.409µs I0331 04:02:39.144127 3904 cni.go:403] [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default] I0331 04:02:39.144345 3904 cni.go:403] [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default] I0331 04:02:39.154247 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 10a76770a9a2612, network: default, NAD default, SandboxID: "10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d", PCI device ID: , UID: "500a76f1-0694-44d1-a0bb-70e245ffa69e", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0331 04:02:39.154347 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 5ff014762c9e825, network: default, NAD default, SandboxID: "5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb", PCI device ID: , UID: "ccea40af-87a1-47ac-a767-8c0218eea4b2", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0331 04:02:39.202628 3904 cni.go:424] [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:58:7a:a3:06:29\",\"name\":\"5ff014762c9e825\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c8efa219-1b38-461f-bcf1-6edf1bcd76aa\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:39.225511 3904 cni.go:424] [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:1c:ca:fb:6d:ed\",\"name\":\"10a76770a9a2612\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/547b10c6-7c37-4fbf-8ea0-e7ebbcff833c\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:40.997855 3904 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0331 04:02:44.072570 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:02:44.074254 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:02:44.074277 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak I0331 04:03:00.172142 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 W0331 04:03:00.969955 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:00.969977 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:00.970068 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0331 04:03:01.602556 3904 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-vt9gh for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:01.602672 3904 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh I0331 04:03:01.616365 3904 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh] pod update took 13.725502ms I0331 04:03:01.616398 3904 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh] addLogicalPort annotation time took 13.763641ms I0331 04:03:01.617784 3904 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh] addLogicalPort took 15.248681ms, libovsdb time 858.177µs I0331 04:03:01.971445 3904 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh d0fb08b945cec031f64d1bdfa702c00685b5f04d22c79ec937c763c678d87d66 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh d0fb08b945cec031f64d1bdfa702c00685b5f04d22c79ec937c763c678d87d66 network default NAD default] I0331 04:03:01.980499 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-vt9gh, hostIfaceName: d0fb08b945cec03, network: default, NAD default, SandboxID: "d0fb08b945cec031f64d1bdfa702c00685b5f04d22c79ec937c763c678d87d66", PCI device ID: , UID: "c0a0dec2-ca1b-4de2-ae59-7ceb5fdb6360", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0331 04:03:02.029779 3904 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh d0fb08b945cec031f64d1bdfa702c00685b5f04d22c79ec937c763c678d87d66 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-vt9gh d0fb08b945cec031f64d1bdfa702c00685b5f04d22c79ec937c763c678d87d66 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:81:1a:4d:30:be\",\"name\":\"d0fb08b945cec03\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/51238dff-5d55-4038-a76c-0365733d054f\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:03:08.213986 3904 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:08.214008 3904 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:08.214015 3904 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0331 04:03:08.545598 3904 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-8tlcq] creating logical port tsf-keycloak_tsf-iam-8tlcq for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:08.545710 3904 kube.go:256] Updating pod tsf-keycloak/tsf-iam-8tlcq I0331 04:03:08.558670 3904 pod.go:62] [tsf-keycloak/tsf-iam-8tlcq] pod update took 12.98909ms I0331 04:03:08.558690 3904 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-8tlcq] addLogicalPort annotation time took 13.019164ms I0331 04:03:08.559954 3904 pods.go:271] [tsf-keycloak/tsf-iam-8tlcq] addLogicalPort took 14.377347ms, libovsdb time 877.276µs I0331 04:03:08.982150 3904 cni.go:403] [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default] I0331 04:03:08.990494 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-8tlcq, hostIfaceName: 781be5d20c21e11, network: default, NAD default, SandboxID: "781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187", PCI device ID: , UID: "4e377982-39ad-434c-887b-4179d12261f5", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0331 04:03:09.038026 3904 cni.go:424] [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:d1:d5:f3:00:66\",\"name\":\"781be5d20c21e11\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21bcfb2a-3ca9-4708-8368-68327863b2c3\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:03:12.256216 3904 cni.go:403] [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default] I0331 04:03:12.337393 3904 cni.go:424] [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 5ff014762c9e825719a843414bb62cc3e98a06fac01d7039b2131902e08f06cb network default NAD default], result "{}", err I0331 04:03:12.423077 3904 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:12.423218 3904 pods.go:176] Deleting pod: tsf/test-tsf-iam I0331 04:03:12.424748 3904 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.44 I0331 04:03:12.424783 3904 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:12.584302 3904 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:12.584335 3904 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:12.589145 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0331 04:03:12.589173 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0331 04:03:13.545421 3904 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:13.545546 3904 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0331 04:03:13.564703 3904 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 19.190719ms I0331 04:03:13.564800 3904 base_network_controller_pods.go:937] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 19.29265ms I0331 04:03:13.567621 3904 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 22.221612ms, libovsdb time 2.331008ms I0331 04:03:13.940089 3904 cni.go:403] [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default] ADD starting CNI request [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default] I0331 04:03:13.981840 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: f0001bbe4112f27, network: default, NAD default, SandboxID: "f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129", PCI device ID: , UID: "f0e6fd2d-e5c2-4ada-9392-99c0d27019ad", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0331 04:03:14.046712 3904 cni.go:424] [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default] ADD finished CNI request [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:90:ce:48:40:7b\",\"name\":\"f0001bbe4112f27\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0010b0d6-f1a4-4c86-9bbb-036e36258ef5\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:03:19.440134 3904 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:03:19.440162 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:03:21.984286 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:21.984319 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:21.984327 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:23.458402 3904 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0331 04:03:25.333328 3904 cni.go:403] [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default] I0331 04:03:25.406545 3904 cni.go:424] [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-8tlcq 781be5d20c21e11a6cdcd420c2e2e7c2cca4588611b964dc2736225cee848187 network default NAD default], result "{}", err I0331 04:03:25.468852 3904 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-8tlcq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:25.468888 3904 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-8tlcq I0331 04:03:25.470133 3904 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-8tlcq, ips: 10.128.0.46 I0331 04:03:25.470158 3904 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-8tlcq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 04:03:26.471470 3904 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0331 04:03:26.493980 3904 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-8tlcq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:26.494002 3904 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-8tlcq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:26.720727 3904 cni.go:403] [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default] I0331 04:03:26.799477 3904 cni.go:424] [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 10a76770a9a2612decbff3d0285a7076c6c70c48dac22ade3e3134bcbcd1768d network default NAD default], result "{}", err I0331 04:03:26.849181 3904 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:26.849289 3904 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0331 04:03:26.850537 3904 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.43 I0331 04:03:26.850566 3904 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:27.339392 3904 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:27.339420 3904 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:27.344049 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0331 04:03:27.344086 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0331 04:03:27.391766 3904 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:27.391852 3904 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0331 04:03:27.406162 3904 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 14.339961ms I0331 04:03:27.406183 3904 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 14.363186ms I0331 04:03:27.407454 3904 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 15.699975ms, libovsdb time 885.961µs W0331 04:03:27.453443 3904 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0331 04:03:27.754454 3904 cni.go:403] [tsf-keycloak/keycloak-0 92c37e7af572ff4ee1356ef1c85436947e37fe236d8b411f684cc8368caf4e1f network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 92c37e7af572ff4ee1356ef1c85436947e37fe236d8b411f684cc8368caf4e1f network default NAD default] I0331 04:03:27.763272 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 92c37e7af572ff4, network: default, NAD default, SandboxID: "92c37e7af572ff4ee1356ef1c85436947e37fe236d8b411f684cc8368caf4e1f", PCI device ID: , UID: "118504a4-ea23-41cf-9917-1cbbf2bc2cb3", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0331 04:03:27.809563 3904 cni.go:424] [tsf-keycloak/keycloak-0 92c37e7af572ff4ee1356ef1c85436947e37fe236d8b411f684cc8368caf4e1f network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 92c37e7af572ff4ee1356ef1c85436947e37fe236d8b411f684cc8368caf4e1f network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:a0:21:71:47:c2\",\"name\":\"92c37e7af572ff4\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3fb178b8-9477-4801-9a90-bd0c84c5bb81\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:03:27.914130 3904 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-kjrxz] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-kjrxz for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:27.914281 3904 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-kjrxz I0331 04:03:27.924799 3904 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-kjrxz] pod update took 10.54109ms I0331 04:03:27.924822 3904 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-kjrxz] addLogicalPort annotation time took 10.567637ms I0331 04:03:27.926181 3904 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-kjrxz] addLogicalPort took 12.063743ms, libovsdb time 900.698µs I0331 04:03:28.275755 3904 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-kjrxz 20ca02e9b356080f51e818f32bdd1310309ded86b83a902848488be3dfc06c51 network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-kjrxz 20ca02e9b356080f51e818f32bdd1310309ded86b83a902848488be3dfc06c51 network default NAD default] I0331 04:03:28.284263 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-kjrxz, hostIfaceName: 20ca02e9b356080, network: default, NAD default, SandboxID: "20ca02e9b356080f51e818f32bdd1310309ded86b83a902848488be3dfc06c51", PCI device ID: , UID: "c44092ea-5353-48fe-8405-82f3b900d03f", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0331 04:03:28.331515 3904 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-kjrxz 20ca02e9b356080f51e818f32bdd1310309ded86b83a902848488be3dfc06c51 network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-kjrxz 20ca02e9b356080f51e818f32bdd1310309ded86b83a902848488be3dfc06c51 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:f3:f1:ac:6f:29\",\"name\":\"20ca02e9b356080\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8fc38199-b75b-4e9f-8966-45e01dbe5873\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:03:29.080542 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=1030 W0331 04:03:29.367087 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:29.367184 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:29.369420 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0331 04:03:30.532082 3904 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:30.532669 3904 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:31.552624 3904 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:31.552721 3904 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:42.156123 3904 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0331 04:03:44.073876 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:03:44.076639 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:44.076659 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak I0331 04:03:52.091116 3904 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-2qmbf for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:52.091246 3904 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf I0331 04:03:52.105933 3904 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf] pod update took 14.741019ms I0331 04:03:52.105959 3904 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf] addLogicalPort annotation time took 14.771936ms I0331 04:03:52.107429 3904 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf] addLogicalPort took 16.334962ms, libovsdb time 943.789µs I0331 04:03:52.292569 3904 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-6brc7 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:03:52.292663 3904 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7 I0331 04:03:52.305512 3904 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7] pod update took 12.87055ms I0331 04:03:52.305539 3904 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7] addLogicalPort annotation time took 12.90109ms I0331 04:03:52.306891 3904 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7] addLogicalPort took 14.340474ms, libovsdb time 955.974µs I0331 04:03:52.453795 3904 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf a66ff0508513a9328817a791b1d3e2909378932014e5bc65f5962272515b9475 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf a66ff0508513a9328817a791b1d3e2909378932014e5bc65f5962272515b9475 network default NAD default] I0331 04:03:52.462416 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-2qmbf, hostIfaceName: a66ff0508513a93, network: default, NAD default, SandboxID: "a66ff0508513a9328817a791b1d3e2909378932014e5bc65f5962272515b9475", PCI device ID: , UID: "029a2e05-133f-416d-bcfa-f4f3f763f4ed", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0331 04:03:52.511711 3904 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf a66ff0508513a9328817a791b1d3e2909378932014e5bc65f5962272515b9475 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2qmbf a66ff0508513a9328817a791b1d3e2909378932014e5bc65f5962272515b9475 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:7b:eb:cc:65:49\",\"name\":\"a66ff0508513a93\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ef8ecc79-03f6-4023-8e31-578cf6c54f40\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:03:52.655441 3904 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7 1666c9f279f51a2cdc3413b81c459dd0cacb508026f32e3b6f3673f6956269dd network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7 1666c9f279f51a2cdc3413b81c459dd0cacb508026f32e3b6f3673f6956269dd network default NAD default] I0331 04:03:52.664977 3904 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-6brc7, hostIfaceName: 1666c9f279f51a2, network: default, NAD default, SandboxID: "1666c9f279f51a2cdc3413b81c459dd0cacb508026f32e3b6f3673f6956269dd", PCI device ID: , UID: "b2e66fa6-2d80-4449-a3c6-e6da1d6e8a9c", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0331 04:03:52.710953 3904 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7 1666c9f279f51a2cdc3413b81c459dd0cacb508026f32e3b6f3673f6956269dd network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-6brc7 1666c9f279f51a2cdc3413b81c459dd0cacb508026f32e3b6f3673f6956269dd network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:77:b0:81:52:a7\",\"name\":\"1666c9f279f51a2\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6bb125f4-0d60-4f46-b8dc-105e24fb97a4\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:03:55.561979 3904 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:03:55.561999 3904 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:03:56.529004 3904 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0331 04:03:56.551854 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:03:56.551877 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0331 04:03:58.177865 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0331 04:04:02.574525 3904 cni.go:403] [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default] DEL starting CNI request [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default] I0331 04:04:02.648009 3904 cni.go:424] [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default] DEL finished CNI request [tsf-tas/test-tsf-tas f0001bbe4112f271fa7bf3365addb96462fa27e542bdb358d666f322b2c41129 network default NAD default], result "{}", err I0331 04:04:02.700601 3904 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:02.700629 3904 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0331 04:04:02.701960 3904 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.47 I0331 04:04:02.701982 3904 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:02.720878 3904 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:02.720892 3904 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:02.726620 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0331 04:04:02.726646 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0331 04:04:08.860737 3904 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:08.861309 3904 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:09.729995 3904 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:09.730018 3904 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:10.863318 3904 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:10.863859 3904 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:11.744577 3904 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:11.744604 3904 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:14.764666 3904 obj_retry.go:464] Detected object tsf/patch-tekton-config-xt9bv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:14.765217 3904 obj_retry.go:464] Detected object tsf/patch-tekton-config-xt9bv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:15.643765 3904 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:15.643793 3904 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0331 04:04:16.765254 3904 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0331 04:04:18.184187 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 W0331 04:04:18.779346 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0331 04:04:19.673049 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0331 04:04:23.071992 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=125 I0331 04:04:24.824617 3904 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:24.825138 3904 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:25.844090 3904 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.844109 3904 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.877502 3904 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.877520 3904 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.882110 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-xt9bv I0331 04:04:25.882136 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-xt9bv W0331 04:04:26.827909 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0331 04:04:36.884917 3904 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:36.885520 3904 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:36.901837 3904 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:36.901858 3904 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:36.909449 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0331 04:04:36.909472 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0331 04:04:37.528769 3904 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-2-188.ec2.internal I0331 04:04:37.528881 3904 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0331 04:04:37.541563 3904 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 12.726496ms I0331 04:04:37.541583 3904 base_network_controller_pods.go:937] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 12.75178ms I0331 04:04:37.542988 3904 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 14.2401ms, libovsdb time 974.181µs I0331 04:04:37.892401 3904 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default] I0331 04:04:37.901347 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 4db9e685ab96de6, network: default, NAD default, SandboxID: "4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428", PCI device ID: , UID: "173f950f-3d8f-494e-a2fa-6d851dddb89c", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0331 04:04:37.948902 3904 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:e0:43:b8:70:58\",\"name\":\"4db9e685ab96de6\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db567207-2e20-434f-a0e3-238d3f083262\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:40.775624 3904 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default] I0331 04:04:40.870980 3904 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 4db9e685ab96de697112777b8f830970f6e7c1d98f9dd8f68bcd60073d049428 network default NAD default], result "{}", err I0331 04:04:40.924315 3904 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:40.924348 3904 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0331 04:04:40.925712 3904 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.52 I0331 04:04:40.925742 3904 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:40.943457 3904 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:40.943546 3904 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:40.949091 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0331 04:04:40.949119 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0331 04:04:41.139223 3904 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-2-188.ec2.internal I0331 04:04:41.139375 3904 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0331 04:04:41.154058 3904 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 14.719435ms I0331 04:04:41.154077 3904 base_network_controller_pods.go:937] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 14.744435ms I0331 04:04:41.155343 3904 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 16.160034ms, libovsdb time 872.464µs I0331 04:04:41.498457 3904 cni.go:403] [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default] ADD starting CNI request [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default] I0331 04:04:41.507947 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 2d08fc5529012e2, network: default, NAD default, SandboxID: "2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3", PCI device ID: , UID: "38b08254-56bf-4ae4-b034-087e8b58dcc8", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0331 04:04:41.554776 3904 cni.go:424] [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:74:b3:31:07:53\",\"name\":\"2d08fc5529012e2\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/72c717c3-4a07-4749-9446-48e3081011bf\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:43.403141 3904 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-gk9fd-9nz9s] creating logical port tsf-tas_tuf-repository-init-gk9fd-9nz9s for pod on switch ip-10-0-2-188.ec2.internal I0331 04:04:43.403311 3904 kube.go:256] Updating pod tsf-tas/tuf-repository-init-gk9fd-9nz9s I0331 04:04:43.416109 3904 pod.go:62] [tsf-tas/tuf-repository-init-gk9fd-9nz9s] pod update took 12.823198ms I0331 04:04:43.416159 3904 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-gk9fd-9nz9s] addLogicalPort annotation time took 12.855896ms I0331 04:04:43.417600 3904 pods.go:271] [tsf-tas/tuf-repository-init-gk9fd-9nz9s] addLogicalPort took 14.48468ms, libovsdb time 885.272µs I0331 04:04:48.985724 3904 obj_retry.go:464] Detected object tsf-tpa/create-db-p8489 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:48.986316 3904 obj_retry.go:464] Detected object tsf-tpa/create-db-p8489 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:49.856787 3904 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:49.856813 3904 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:52.764434 3904 cni.go:403] [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default] I0331 04:04:52.773251 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-gk9fd-9nz9s, hostIfaceName: 35f8ffff20d5249, network: default, NAD default, SandboxID: "35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661", PCI device ID: , UID: "4d584603-efc5-41c3-8b81-ba83363a18be", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0331 04:04:52.820443 3904 cni.go:424] [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:72:5b:e1:21:f9\",\"name\":\"35f8ffff20d5249\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/04c35ab1-548d-4ea7-8411-7f08355e0435\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:53.006115 3904 obj_retry.go:464] Detected object tsf-tpa/migrate-db-q4jqv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:53.006699 3904 obj_retry.go:464] Detected object tsf-tpa/migrate-db-q4jqv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:54.028269 3904 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:54.028291 3904 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:54.206975 3904 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-n6cv9] creating logical port tsf-tpa_create-importers-n6cv9 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:04:54.207063 3904 kube.go:256] Updating pod tsf-tpa/create-importers-n6cv9 I0331 04:04:54.219485 3904 pod.go:62] [tsf-tpa/create-importers-n6cv9] pod update took 12.439462ms I0331 04:04:54.219509 3904 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-n6cv9] addLogicalPort annotation time took 12.467838ms I0331 04:04:54.221007 3904 pods.go:271] [tsf-tpa/create-importers-n6cv9] addLogicalPort took 14.04232ms, libovsdb time 942.57µs I0331 04:04:54.567021 3904 cni.go:403] [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default] I0331 04:04:54.576265 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-n6cv9, hostIfaceName: 4ff6f4342a9b3b6, network: default, NAD default, SandboxID: "4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac", PCI device ID: , UID: "3c45f831-6bc0-4228-9f84-184aa612d23b", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0331 04:04:54.623440 3904 cni.go:424] [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:d4:89:91:04:04\",\"name\":\"4ff6f4342a9b3b6\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7bbe1c80-ce3a-46d6-88ed-e7c318b0f424\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:56.882658 3904 cni.go:403] [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default] I0331 04:04:56.962864 3904 cni.go:424] [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-gk9fd-9nz9s 35f8ffff20d524995d665998e6cb05d156b9d861fc8030ff0b3af5c44eccb661 network default NAD default], result "{}", err I0331 04:04:57.075802 3904 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:57.075831 3904 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-gk9fd-9nz9s I0331 04:04:57.077319 3904 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-gk9fd-9nz9s, ips: 10.128.0.54 I0331 04:04:57.077349 3904 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:57.893207 3904 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:57.893230 3904 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:58.893080 3904 cni.go:403] [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default] I0331 04:04:58.961668 3904 cni.go:424] [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-n6cv9 4ff6f4342a9b3b619cdc277183734e77e6bd6019a6daa0fe4edf883224aff2ac network default NAD default], result "{}", err I0331 04:04:59.009538 3904 obj_retry.go:464] Detected object tsf-tpa/create-importers-n6cv9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:59.009568 3904 pods.go:176] Deleting pod: tsf-tpa/create-importers-n6cv9 I0331 04:04:59.010873 3904 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-n6cv9, ips: 10.128.0.55 I0331 04:04:59.010900 3904 obj_retry.go:464] Detected object tsf-tpa/create-importers-n6cv9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:00.033810 3904 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:00.033837 3904 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:00.517084 3904 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-pxjgj] creating logical port tsf-tas_tuf-bc5768c88-pxjgj for pod on switch ip-10-0-2-188.ec2.internal I0331 04:05:00.517189 3904 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-pxjgj I0331 04:05:00.535351 3904 pod.go:62] [tsf-tas/tuf-bc5768c88-pxjgj] pod update took 18.185931ms I0331 04:05:00.535375 3904 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-pxjgj] addLogicalPort annotation time took 18.212402ms I0331 04:05:00.536583 3904 pods.go:271] [tsf-tas/tuf-bc5768c88-pxjgj] addLogicalPort took 19.522277ms, libovsdb time 822.009µs I0331 04:05:02.975126 3904 cni.go:403] [tsf-tas/tuf-bc5768c88-pxjgj 9658935b32e438e4a395cd81a17a4594faa2050fe292a818d0087f3bc240d58d network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-pxjgj 9658935b32e438e4a395cd81a17a4594faa2050fe292a818d0087f3bc240d58d network default NAD default] I0331 04:05:02.984142 3904 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-pxjgj, hostIfaceName: 9658935b32e438e, network: default, NAD default, SandboxID: "9658935b32e438e4a395cd81a17a4594faa2050fe292a818d0087f3bc240d58d", PCI device ID: , UID: "2f3aa835-6ee4-485b-8573-e9faae644473", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0331 04:05:03.032708 3904 cni.go:424] [tsf-tas/tuf-bc5768c88-pxjgj 9658935b32e438e4a395cd81a17a4594faa2050fe292a818d0087f3bc240d58d network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-pxjgj 9658935b32e438e4a395cd81a17a4594faa2050fe292a818d0087f3bc240d58d network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:0e:b9:98:de:f2\",\"name\":\"9658935b32e438e\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0eefd3c5-725e-4add-8764-06878cb1f2a5\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:05:05.930589 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0331 04:05:08.985025 3904 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0331 04:05:16.942060 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0331 04:05:35.092411 3904 cni.go:403] [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default] DEL starting CNI request [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default] I0331 04:05:35.162672 3904 cni.go:424] [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 2d08fc5529012e27c40ac5bd223c62201cf0064bdff5fb996eabfb1766f0b3f3 network default NAD default], result "{}", err I0331 04:05:35.214823 3904 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:35.214852 3904 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0331 04:05:35.216339 3904 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.53 I0331 04:05:35.216365 3904 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:35.253219 3904 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:35.253244 3904 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:35.260115 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0331 04:05:35.260134 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0331 04:05:35.931103 3904 namespace.go:98] [build-service] adding namespace I0331 04:05:35.931146 3904 namespace.go:98] [integration-service] adding namespace I0331 04:05:35.931151 3904 namespace.go:98] [image-controller] adding namespace I0331 04:05:35.932075 3904 namespace.go:102] [build-service] adding namespace took 950.147µs I0331 04:05:35.932743 3904 namespace.go:102] [integration-service] adding namespace took 1.582933ms I0331 04:05:35.933338 3904 namespace.go:102] [image-controller] adding namespace took 2.17424ms I0331 04:05:35.963115 3904 namespace.go:142] [build-service] updating namespace I0331 04:05:35.983765 3904 namespace.go:142] [build-service] updating namespace I0331 04:05:35.999963 3904 namespace.go:142] [integration-service] updating namespace I0331 04:05:36.022925 3904 namespace.go:142] [integration-service] updating namespace I0331 04:05:36.026580 3904 namespace.go:142] [image-controller] updating namespace I0331 04:05:36.045339 3904 namespace.go:142] [image-controller] updating namespace I0331 04:05:36.166097 3904 namespace.go:142] [build-service] updating namespace I0331 04:05:36.186914 3904 namespace.go:142] [integration-service] updating namespace I0331 04:05:36.255042 3904 namespace.go:98] [release-service] adding namespace I0331 04:05:36.255896 3904 namespace.go:102] [release-service] adding namespace took 828.486µs I0331 04:05:36.264660 3904 namespace.go:142] [konflux-ui] updating namespace I0331 04:05:36.284795 3904 namespace.go:142] [release-service] updating namespace I0331 04:05:36.321241 3904 namespace.go:142] [release-service] updating namespace I0331 04:05:36.370752 3904 namespace.go:98] [konflux-info] adding namespace I0331 04:05:36.371569 3904 namespace.go:102] [konflux-info] adding namespace took 798.031µs I0331 04:05:36.448880 3904 namespace.go:142] [konflux-info] updating namespace I0331 04:05:36.472994 3904 namespace.go:98] [namespace-lister] adding namespace I0331 04:05:36.473139 3904 namespace.go:98] [enterprise-contract-service] adding namespace I0331 04:05:36.473760 3904 namespace.go:102] [namespace-lister] adding namespace took 749.507µs I0331 04:05:36.474308 3904 namespace.go:102] [enterprise-contract-service] adding namespace took 1.154051ms I0331 04:05:36.491010 3904 namespace.go:142] [konflux-info] updating namespace I0331 04:05:36.502042 3904 namespace.go:142] [image-controller] updating namespace I0331 04:05:36.538072 3904 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:36.607846 3904 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:36.616821 3904 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:36.639725 3904 namespace.go:98] [default-tenant] adding namespace I0331 04:05:36.640507 3904 namespace.go:102] [default-tenant] adding namespace took 765.161µs I0331 04:05:36.689872 3904 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:36.702821 3904 namespace.go:98] [segment-bridge] adding namespace I0331 04:05:36.703608 3904 namespace.go:102] [segment-bridge] adding namespace took 771.586µs I0331 04:05:36.737840 3904 namespace.go:142] [default-tenant] updating namespace I0331 04:05:36.780980 3904 namespace.go:142] [default-tenant] updating namespace I0331 04:05:36.847146 3904 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:36.881928 3904 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:37.059075 3904 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0331 04:05:37.059699 3904 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0331 04:05:37.059718 3904 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0331 04:05:37.062418 3904 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0331 04:05:37.193497 3904 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0331 04:05:37.193523 3904 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0331 04:05:37.194393 3904 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0331 04:05:37.681061 3904 namespace.go:142] [build-service] updating namespace I0331 04:05:38.318273 3904 namespace.go:142] [image-controller] updating namespace I0331 04:05:38.358427 3904 base_network_controller_pods.go:487] [default/konflux-ui/dex-77d4fd78c-cmtwq] creating logical port konflux-ui_dex-77d4fd78c-cmtwq for pod on switch ip-10-0-2-188.ec2.internal I0331 04:05:38.358535 3904 kube.go:256] Updating pod konflux-ui/dex-77d4fd78c-cmtwq I0331 04:05:38.417951 3904 pod.go:62] [konflux-ui/dex-77d4fd78c-cmtwq] pod update took 59.443262ms I0331 04:05:38.417979 3904 base_network_controller_pods.go:937] [default/konflux-ui/dex-77d4fd78c-cmtwq] addLogicalPort annotation time took 59.476175ms I0331 04:05:38.419417 3904 pods.go:271] [konflux-ui/dex-77d4fd78c-cmtwq] addLogicalPort took 61.012769ms, libovsdb time 836.609µs I0331 04:05:38.475996 3904 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-tcdcx for pod on switch ip-10-0-2-188.ec2.internal I0331 04:05:38.476134 3904 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx I0331 04:05:38.532349 3904 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx] pod update took 56.276987ms I0331 04:05:38.532440 3904 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx] addLogicalPort annotation time took 56.372851ms I0331 04:05:38.533945 3904 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx] addLogicalPort took 57.961856ms, libovsdb time 949.336µs I0331 04:05:38.766399 3904 namespace.go:142] [integration-service] updating namespace I0331 04:05:38.824904 3904 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default] I0331 04:05:38.833716 3904 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-tcdcx, hostIfaceName: 60f60da7cb407b2, network: default, NAD default, SandboxID: "60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68", PCI device ID: , UID: "cef4f673-0ddb-4c75-8258-08b605917474", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0331 04:05:39.056226 3904 namespace.go:142] [konflux-info] updating namespace I0331 04:05:39.089060 3904 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:f2:a0:11:48:6c\",\"name\":\"60f60da7cb407b2\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b332478f-ce88-4a55-bc4d-5a8c19ba411a\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:05:39.253918 3904 namespace.go:142] [release-service] updating namespace I0331 04:05:39.279373 3904 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:39.279919 3904 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:39.385910 3904 namespace.go:142] [build-service] updating namespace I0331 04:05:39.397122 3904 namespace.go:142] [image-controller] updating namespace I0331 04:05:39.414479 3904 namespace.go:142] [integration-service] updating namespace I0331 04:05:39.452021 3904 namespace.go:142] [konflux-info] updating namespace I0331 04:05:39.471286 3904 namespace.go:142] [release-service] updating namespace I0331 04:05:39.500236 3904 base_network_controller_pods.go:487] [default/konflux-ui/proxy-59b64cb865-6g7z5] creating logical port konflux-ui_proxy-59b64cb865-6g7z5 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:05:39.500332 3904 kube.go:256] Updating pod konflux-ui/proxy-59b64cb865-6g7z5 I0331 04:05:39.525476 3904 pod.go:62] [konflux-ui/proxy-59b64cb865-6g7z5] pod update took 25.176321ms I0331 04:05:39.525502 3904 base_network_controller_pods.go:937] [default/konflux-ui/proxy-59b64cb865-6g7z5] addLogicalPort annotation time took 25.204599ms I0331 04:05:39.526936 3904 pods.go:271] [konflux-ui/proxy-59b64cb865-6g7z5] addLogicalPort took 26.715648ms, libovsdb time 897.51µs I0331 04:05:39.545423 3904 namespace.go:142] [build-service] updating namespace I0331 04:05:39.616326 3904 namespace.go:142] [image-controller] updating namespace I0331 04:05:39.684145 3904 namespace.go:142] [integration-service] updating namespace I0331 04:05:39.803594 3904 namespace.go:142] [konflux-info] updating namespace I0331 04:05:40.008567 3904 namespace.go:142] [release-service] updating namespace I0331 04:05:40.110295 3904 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7669cd69b5-ws5w9] creating logical port integration-service_integration-service-controller-manager-7669cd69b5-ws5w9 for pod on switch ip-10-0-2-188.ec2.internal I0331 04:05:40.110383 3904 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7669cd69b5-ws5w9 I0331 04:05:40.126556 3904 pod.go:62] [integration-service/integration-service-controller-manager-7669cd69b5-ws5w9] pod update took 16.19841ms I0331 04:05:40.126576 3904 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-7669cd69b5-ws5w9] addLogicalPort annotation time took 16.222992ms I0331 04:05:40.127834 3904 pods.go:271] [integration-service/integration-service-controller-manager-7669cd69b5-ws5w9] addLogicalPort took 17.557685ms, libovsdb time 854.19µs W0331 04:05:40.244998 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0331 04:05:40.305536 3904 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:40.305555 3904 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:40.497154 3904 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-188.ec2.internal I0331 04:05:40.497284 3904 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0331 04:05:40.528863 3904 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 31.611338ms I0331 04:05:40.528888 3904 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 31.640175ms I0331 04:05:40.530164 3904 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 33.030641ms, libovsdb time 829.581µs I0331 04:05:40.862444 3904 cni.go:403] [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default] I0331 04:05:40.871817 3904 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: cea5d4d7c9c0726, network: default, NAD default, SandboxID: "cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845", PCI device ID: , UID: "8ff4f2c5-0261-46f1-ba1d-7f54ae7fd727", MAC: 0a:58:0a:80:00:3d, IPs: [10.128.0.61/23] I0331 04:05:40.925561 3904 cni.go:424] [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:10:75:27:94:8f\",\"name\":\"cea5d4d7c9c0726\"},{\"mac\":\"0a:58:0a:80:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4276259-6d0f-4256-bc32-f60ca7906dcb\"}],\"ips\":[{\"address\":\"10.128.0.61/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:05:41.161803 3904 namespace.go:142] [default-tenant] updating namespace I0331 04:05:41.199778 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0331 04:05:41.362677 3904 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:41.562219 3904 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:41.767571 3904 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:41.884429 3904 namespace.go:142] [default-tenant] updating namespace I0331 04:05:41.896527 3904 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:41.909646 3904 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:41.921492 3904 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:41.993345 3904 namespace.go:142] [default-tenant] updating namespace I0331 04:05:42.095569 3904 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:42.191705 3904 namespace.go:142] [namespace-lister] updating namespace W0331 04:05:42.244343 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0331 04:05:42.293216 3904 namespace.go:142] [segment-bridge] updating namespace W0331 04:05:43.142234 3904 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0331 04:05:49.258479 3904 cni.go:403] [konflux-ui/dex-77d4fd78c-cmtwq 41b4119b186accc0669e89dc9a843ade7f6e8608bba82c916ee41ab009ea4a04 network default NAD default] ADD starting CNI request [konflux-ui/dex-77d4fd78c-cmtwq 41b4119b186accc0669e89dc9a843ade7f6e8608bba82c916ee41ab009ea4a04 network default NAD default] I0331 04:05:49.263706 3904 cni.go:403] [integration-service/integration-service-controller-manager-7669cd69b5-ws5w9 e3475396d40844dc248302467aa0fb36d72dd6ecf73e5c08c1a350d8a7948aaa network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-7669cd69b5-ws5w9 e3475396d40844dc248302467aa0fb36d72dd6ecf73e5c08c1a350d8a7948aaa network default NAD default] I0331 04:05:49.264861 3904 cni.go:403] [konflux-ui/proxy-59b64cb865-6g7z5 a0eb1e7a6d84459e8236bda877a7efa015d29a687fb5e33a236f20733553d1bb network default NAD default] ADD starting CNI request [konflux-ui/proxy-59b64cb865-6g7z5 a0eb1e7a6d84459e8236bda877a7efa015d29a687fb5e33a236f20733553d1bb network default NAD default] I0331 04:05:49.267329 3904 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-77d4fd78c-cmtwq, hostIfaceName: 41b4119b186accc, network: default, NAD default, SandboxID: "41b4119b186accc0669e89dc9a843ade7f6e8608bba82c916ee41ab009ea4a04", PCI device ID: , UID: "8f4a3fee-5bfc-4d6f-a6d0-b853ea2e7b57", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0331 04:05:49.276086 3904 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-59b64cb865-6g7z5, hostIfaceName: a0eb1e7a6d84459, network: default, NAD default, SandboxID: "a0eb1e7a6d84459e8236bda877a7efa015d29a687fb5e33a236f20733553d1bb", PCI device ID: , UID: "66169208-bfc6-4cd8-84de-898fa6c00dcc", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0331 04:05:49.276648 3904 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7669cd69b5-ws5w9, hostIfaceName: e3475396d40844d, network: default, NAD default, SandboxID: "e3475396d40844dc248302467aa0fb36d72dd6ecf73e5c08c1a350d8a7948aaa", PCI device ID: , UID: "315f27b6-cf38-4c1a-9ca1-29b490a4fa66", MAC: 0a:58:0a:80:00:3c, IPs: [10.128.0.60/23] I0331 04:05:49.325481 3904 cni.go:424] [konflux-ui/dex-77d4fd78c-cmtwq 41b4119b186accc0669e89dc9a843ade7f6e8608bba82c916ee41ab009ea4a04 network default NAD default] ADD finished CNI request [konflux-ui/dex-77d4fd78c-cmtwq 41b4119b186accc0669e89dc9a843ade7f6e8608bba82c916ee41ab009ea4a04 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:1a:12:d7:35:9b\",\"name\":\"41b4119b186accc\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c1dfede4-5d32-4849-b36e-b9d862488c26\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:05:49.552492 3904 cni.go:424] [integration-service/integration-service-controller-manager-7669cd69b5-ws5w9 e3475396d40844dc248302467aa0fb36d72dd6ecf73e5c08c1a350d8a7948aaa network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-7669cd69b5-ws5w9 e3475396d40844dc248302467aa0fb36d72dd6ecf73e5c08c1a350d8a7948aaa network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:b8:e5:8f:74:80\",\"name\":\"e3475396d40844d\"},{\"mac\":\"0a:58:0a:80:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d294d5a7-add4-4983-8052-cea0e35b6556\"}],\"ips\":[{\"address\":\"10.128.0.60/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:05:49.552525 3904 cni.go:424] [konflux-ui/proxy-59b64cb865-6g7z5 a0eb1e7a6d84459e8236bda877a7efa015d29a687fb5e33a236f20733553d1bb network default NAD default] ADD finished CNI request [konflux-ui/proxy-59b64cb865-6g7z5 a0eb1e7a6d84459e8236bda877a7efa015d29a687fb5e33a236f20733553d1bb network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:0f:a1:2e:1e:03\",\"name\":\"a0eb1e7a6d84459\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d4f310d-7086-45d6-ad3a-aa0309c12884\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:05:53.324917 3904 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0331 04:05:53.325621 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0331 04:05:54.157765 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0331 04:05:54.341584 3904 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:54.341613 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:54.361519 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:05:54.362752 3904 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:05:55.240561 3904 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:05:55.240669 3904 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:05:55.267054 3904 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:55.267070 3904 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:56.273004 3904 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0331 04:06:05.194552 3904 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 W0331 04:06:05.345296 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:06:05.345322 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:06:05.352055 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:06:05.352081 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:06:06.254508 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:06:06.254636 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0331 04:06:06.564304 3904 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:06.564853 3904 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:07.415658 3904 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:07.415677 3904 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:07.421737 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-btztn I0331 04:06:07.421760 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-btztn W0331 04:06:08.267482 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0331 04:06:31.398999 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:31.399078 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:31.399102 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:31.399110 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:33.325850 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:33.325875 3904 util.go:959] Failed to build node endpoints for node ip-10-0-2-188.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0331 04:06:47.075860 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=163 I0331 04:07:03.461529 3904 obj_retry.go:464] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:03.462104 3904 obj_retry.go:464] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:03.732118 3904 obj_retry.go:458] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:03.732138 3904 obj_retry.go:458] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:03.738047 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-646d664c7b-8jppf I0331 04:07:03.738072 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-646d664c7b-8jppf I0331 04:07:08.076792 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0331 04:07:11.721936 3904 cni.go:403] [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default] I0331 04:07:11.810398 3904 cni.go:424] [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux cea5d4d7c9c0726e6482d39c6375915cb01af4260ba4bc7d8c13391c3537c845 network default NAD default], result "{}", err I0331 04:07:11.861949 3904 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:11.861984 3904 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0331 04:07:11.863509 3904 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.61 I0331 04:07:11.863545 3904 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:11.878245 3904 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:11.878328 3904 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:11.886868 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0331 04:07:11.886900 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0331 04:07:39.926590 3904 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-vh7xh] creating logical port image-controller_image-controller-controller-manager-56b684c87-vh7xh for pod on switch ip-10-0-2-188.ec2.internal I0331 04:07:39.926706 3904 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-vh7xh I0331 04:07:39.950554 3904 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-vh7xh] pod update took 23.876227ms I0331 04:07:39.950576 3904 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-vh7xh] addLogicalPort annotation time took 23.90509ms I0331 04:07:39.951822 3904 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-vh7xh] addLogicalPort took 25.25053ms, libovsdb time 858.926µs I0331 04:07:40.222684 3904 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:40.223279 3904 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:40.289782 3904 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-vh7xh c85b6f564e66308679bd0dcf7dc0d15ba98aa1ca7578ab4ff7df899214a96f0f network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-vh7xh c85b6f564e66308679bd0dcf7dc0d15ba98aa1ca7578ab4ff7df899214a96f0f network default NAD default] I0331 04:07:40.299234 3904 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-vh7xh, hostIfaceName: c85b6f564e66308, network: default, NAD default, SandboxID: "c85b6f564e66308679bd0dcf7dc0d15ba98aa1ca7578ab4ff7df899214a96f0f", PCI device ID: , UID: "bdcf8b3e-6917-4468-bc67-65cb7d5c4f7e", MAC: 0a:58:0a:80:00:3e, IPs: [10.128.0.62/23] I0331 04:07:40.348283 3904 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-vh7xh c85b6f564e66308679bd0dcf7dc0d15ba98aa1ca7578ab4ff7df899214a96f0f network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-vh7xh c85b6f564e66308679bd0dcf7dc0d15ba98aa1ca7578ab4ff7df899214a96f0f network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:a4:57:88:2f:43\",\"name\":\"c85b6f564e66308\"},{\"mac\":\"0a:58:0a:80:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f074cdf4-8a9d-4495-b27f-a5469b3826da\"}],\"ips\":[{\"address\":\"10.128.0.62/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:07:40.943600 3904 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:40.943622 3904 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:40.949039 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb I0331 04:07:40.949070 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb I0331 04:07:53.080813 3904 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default] I0331 04:07:53.156669 3904 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx 60f60da7cb407b2abfa2f2c8d50798e8b595bff280acb59178b30c80feeccf68 network default NAD default], result "{}", err I0331 04:07:53.209438 3904 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:53.209472 3904 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx I0331 04:07:53.210899 3904 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx, ips: 10.128.0.58 I0331 04:07:53.210933 3904 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:53.961552 3904 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:53.961581 3904 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:53.967133 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx I0331 04:07:53.967164 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx I0331 04:08:48.158923 3904 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0331 04:09:17.159232 3904 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0331 04:09:28.170256 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0331 04:09:29.075852 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=158 I0331 04:10:03.860032 3904 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:03.860692 3904 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:04.727599 3904 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:04.727644 3904 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:38.083768 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=321 I0331 04:10:43.181486 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0331 04:10:47.199433 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0331 04:10:58.202883 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0331 04:11:37.198544 3904 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0331 04:11:42.075306 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=51 I0331 04:11:44.163008 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0331 04:11:56.075041 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=53 I0331 04:12:39.175340 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0331 04:13:11.187906 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0331 04:14:15.079964 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0331 04:14:29.078843 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=5 I0331 04:14:49.045973 3904 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:49.046023 3904 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:49.050586 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-p8489 I0331 04:14:49.050604 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-p8489 I0331 04:14:54.031132 3904 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:54.031178 3904 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:54.033955 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-q4jqv I0331 04:14:54.033972 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-q4jqv I0331 04:15:00.028651 3904 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:00.028708 3904 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:00.032617 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-n6cv9 I0331 04:15:00.032643 3904 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-n6cv9 I0331 04:15:05.079789 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=17 I0331 04:15:55.162355 3904 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0331 04:17:10.163402 3904 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0331 04:18:05.202396 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:18:07.165518 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0331 04:18:12.191391 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0331 04:18:19.206344 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0331 04:18:25.086797 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=14 I0331 04:19:25.173461 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0331 04:20:00.202338 3904 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0331 04:20:03.078551 3904 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=57 W0331 04:20:04.483876 3904 duration.go:461] Config duration recorder: measurement expired for pod/kube-system/ecr-credential-refresh-29582160-x57cm I0331 04:20:05.119071 3904 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:05.119680 3904 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:06.139493 3904 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:06.139517 3904 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:14.185104 3904 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11