+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-65.ec2.internal ++ [[ -n ip-10-0-2-65.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-65.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0326 23:34:25.382838898 - disable conntrack on geneve port' I0326 23:34:25.382838898 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0326 23:34:25.425783151 - starting ovnkube-node + echo 'I0326 23:34:25.425783151 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-65.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-65.ec2.internal --init-node ip-10-0-2-65.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-65.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0326 23:34:25.465628 4021 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0326 23:34:25.465674 4021 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-41e5d74f87.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0326 23:34:25.467127 4021 kube.go:419] Waiting for certificate I0326 23:34:25.467164 4021 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0326 23:34:25.467208 4021 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0326 23:34:25.467996 4021 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:34:25.498093 4021 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:34:25.498124 4021 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:34:25.501100 4021 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:34:25.501186 4021 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9klg6" I0326 23:34:25.510110 4021 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9klg6" I0326 23:34:25.510186 4021 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:34:26.468196 4021 kube.go:426] Certificate found I0326 23:34:26.468891 4021 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:34:26.470128 4021 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0326 23:34:26.526035 4021 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-27 23:29:25 +0000 UTC" deadline="2026-03-27 20:06:42.949536779 +0000 UTC" I0326 23:34:26.526073 4021 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h32m16.423476517s" I0326 23:34:26.536851 4021 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:34:26.546166 4021 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0326 23:34:26.548559 4021 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0326 23:34:26.548700 4021 factory.go:531] Starting watch factory I0326 23:34:26.549088 4021 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549148 4021 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549212 4021 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549247 4021 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549708 4021 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549735 4021 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549932 4021 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.549951 4021 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.550406 4021 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.550473 4021 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.551665 4021 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.551691 4021 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.556540 4021 controller_manager.go:368] Starting the ovnkube controller I0326 23:34:26.556556 4021 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-65.ec2.internal I0326 23:34:26.556627 4021 controller_manager.go:393] NBDB zone sync took: 61.387µs I0326 23:34:26.556638 4021 factory.go:531] Starting watch factory I0326 23:34:26.572736 4021 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.573055 4021 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.573089 4021 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.573162 4021 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.573634 4021 ovn_northd.go:121] Registering OVN northd metrics I0326 23:34:26.573682 4021 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0326 23:34:26.574211 4021 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.578119 4021 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0326 23:34:26.578232 4021 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0326 23:34:26.583067 4021 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:34:26.594220 4021 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0326 23:34:26.655388 4021 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:34:26.655413 4021 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:34:26.655417 4021 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:34:26.655420 4021 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:34:26.655423 4021 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:34:26.655426 4021 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:34:26.655536 4021 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:34:26.655565 4021 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:34:26.655535 4021 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:34:26.655731 4021 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:34:26.658751 4021 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:34:26.658753 4021 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:34:26.660482 4021 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:34:26.660496 4021 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:34:26.660562 4021 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.660584 4021 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.662471 4021 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:34:26.662481 4021 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:34:26.662485 4021 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:34:26.662488 4021 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:34:26.662491 4021 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:34:26.662494 4021 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:34:26.663280 4021 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.666523 4021 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:34:26.666624 4021 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.666634 4021 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.667968 4021 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:34:26.667980 4021 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:34:26.669153 4021 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.672368 4021 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:34:26.672434 4021 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.672450 4021 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.673673 4021 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:34:26.675262 4021 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.677437 4021 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:34:26.677496 4021 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.677509 4021 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.679693 4021 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:34:26.679907 4021 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.683117 4021 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:34:26.683172 4021 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.683183 4021 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.685335 4021 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:34:26.685657 4021 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.689021 4021 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:34:26.689167 4021 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:34:26.689178 4021 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:34:26.691381 4021 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:34:26.691662 4021 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:34:26.694985 4021 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:34:26.695056 4021 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.695068 4021 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.695105 4021 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.695124 4021 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.696476 4021 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:34:26.697692 4021 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.697695 4021 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:34:26.700178 4021 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:34:26.700193 4021 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:34:26.700204 4021 factory.go:636] Watch Factory start up complete, took: 151.507955ms I0326 23:34:26.700332 4021 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0326 23:34:26.701661 4021 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:34:26.706932 4021 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:34:26.706942 4021 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:34:26.706949 4021 factory.go:636] Watch Factory start up complete, took: 150.311532ms I0326 23:34:26.706957 4021 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-65.ec2.internal" zone I0326 23:34:26.760457 4021 default_node_network_controller.go:709] Initializing the default node network controller I0326 23:34:26.823463 4021 udn_isolation.go:101] Starting UDN host isolation manager I0326 23:34:26.829592 4021 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0326 23:34:26.878372 4021 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0326 23:34:26.878454 4021 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0326 23:34:26.878470 4021 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0326 23:34:26.912107 4021 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0326 23:34:26.912174 4021 default_node_network_controller.go:808] Node ip-10-0-2-65.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0326 23:34:26.917715 4021 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.65"] k8s.ovn.org/zone-name:ip-10-0-2-65.ec2.internal] on node ip-10-0-2-65.ec2.internal I0326 23:34:26.943994 4021 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0326 23:34:26.949608 4021 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0326 23:34:26.949742 4021 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0326 23:34:26.949862 4021 gateway_init.go:263] Preparing Gateway I0326 23:34:26.949873 4021 gateway_shared_intf.go:1658] Creating new gateway I0326 23:34:26.957392 4021 controller_manager.go:427] Waiting for node in zone sync took: 250.422115ms I0326 23:34:26.961478 4021 controller_manager.go:608] Switching to transit router for layer2 networks I0326 23:34:26.961492 4021 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-65.ec2.internal I0326 23:34:26.976627 4021 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:34:26.982011 4021 controller_manager.go:288] SCTP support detected in OVN I0326 23:34:26.982593 4021 services_controller.go:69] Creating services controller for network=default I0326 23:34:26.982649 4021 udn_enabled_svc.go:80] Waiting for service informer to sync I0326 23:34:26.982746 4021 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0326 23:34:26.982779 4021 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0326 23:34:26.982794 4021 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0326 23:34:26.982804 4021 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:34:26.982959 4021 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0326 23:34:26.983012 4021 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0326 23:34:26.983040 4021 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 44.769µs I0326 23:34:26.983047 4021 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 24.349µs I0326 23:34:26.983056 4021 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 44.253µs I0326 23:34:26.983063 4021 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0326 23:34:26.983074 4021 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0326 23:34:26.983089 4021 default_network_controller.go:321] Starting the default network controller I0326 23:34:26.983101 4021 udn_enabled_svc.go:95] Performing full resync I0326 23:34:26.983101 4021 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 12.321µs I0326 23:34:26.983383 4021 udn_enabled_svc.go:99] Waiting for handler to sync I0326 23:34:26.983394 4021 udn_enabled_svc.go:104] Starting worker I0326 23:34:26.991202 4021 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0326 23:34:26.991262 4021 repair.go:33] Syncing exgw routes took 41.368µs I0326 23:34:26.991280 4021 default_network_controller.go:407] Starting all the Watchers... I0326 23:34:26.991467 4021 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0326 23:34:26.991481 4021 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0326 23:34:26.991483 4021 namespace.go:98] [openshift-console] adding namespace I0326 23:34:26.991467 4021 namespace.go:98] [openshift-controller-manager] adding namespace I0326 23:34:26.991491 4021 namespace.go:98] [openshift-dns-operator] adding namespace I0326 23:34:26.991491 4021 namespace.go:98] [openshift-insights] adding namespace I0326 23:34:26.991503 4021 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0326 23:34:26.991511 4021 namespace.go:98] [openshift-machine-config-operator] adding namespace I0326 23:34:26.991513 4021 namespace.go:98] [openshift-route-controller-manager] adding namespace I0326 23:34:26.991513 4021 namespace.go:98] [openshift-marketplace] adding namespace I0326 23:34:26.991515 4021 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0326 23:34:26.991531 4021 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0326 23:34:26.991531 4021 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0326 23:34:26.991559 4021 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0326 23:34:26.991470 4021 namespace.go:98] [kube-public] adding namespace I0326 23:34:26.992090 4021 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 611.441µs I0326 23:34:26.992105 4021 namespace.go:98] [openshift-kube-scheduler] adding namespace I0326 23:34:26.992545 4021 namespace.go:102] [openshift-kube-scheduler] adding namespace took 435.006µs I0326 23:34:26.992557 4021 namespace.go:98] [openshift-network-diagnostics] adding namespace W0326 23:34:26.992599 4021 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rz6dg: pod openshift-network-diagnostics/network-check-target-rz6dg: no pod IPs found W0326 23:34:26.992609 4021 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-xm5zg: pod openshift-network-diagnostics/network-check-target-xm5zg: no pod IPs found I0326 23:34:26.993151 4021 namespace.go:102] [openshift-network-diagnostics] adding namespace took 588.871µs I0326 23:34:26.993165 4021 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0326 23:34:26.993761 4021 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 2.27301ms I0326 23:34:26.993771 4021 namespace.go:98] [openshift-service-ca-operator] adding namespace I0326 23:34:26.994355 4021 namespace.go:102] [openshift-console] adding namespace took 2.849384ms I0326 23:34:26.994373 4021 namespace.go:98] [openshift-dns] adding namespace I0326 23:34:26.994932 4021 namespace.go:102] [openshift-controller-manager] adding namespace took 3.435459ms I0326 23:34:26.994948 4021 namespace.go:98] [openshift-package-operator] adding namespace I0326 23:34:26.995463 4021 namespace.go:102] [openshift-dns-operator] adding namespace took 3.964712ms I0326 23:34:26.995475 4021 namespace.go:98] [openshift] adding namespace I0326 23:34:26.995965 4021 namespace.go:102] [openshift-insights] adding namespace took 4.464648ms I0326 23:34:26.995974 4021 namespace.go:98] [openshift-logging] adding namespace I0326 23:34:26.996434 4021 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 4.923677ms I0326 23:34:26.996454 4021 namespace.go:98] [openshift-apiserver-operator] adding namespace I0326 23:34:26.996902 4021 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.383658ms I0326 23:34:26.996915 4021 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0326 23:34:26.997351 4021 namespace.go:102] [openshift-route-controller-manager] adding namespace took 5.82128ms I0326 23:34:26.997365 4021 namespace.go:98] [default] adding namespace I0326 23:34:26.997803 4021 namespace.go:102] [openshift-marketplace] adding namespace took 6.273072ms I0326 23:34:26.997813 4021 namespace.go:98] [dedicated-admin] adding namespace I0326 23:34:26.998251 4021 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.716033ms I0326 23:34:26.998269 4021 namespace.go:98] [openshift-ingress] adding namespace I0326 23:34:26.998716 4021 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.178658ms I0326 23:34:26.998729 4021 namespace.go:98] [openshift-authentication-operator] adding namespace I0326 23:34:26.999147 4021 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.606616ms I0326 23:34:26.999163 4021 namespace.go:98] [openshift-backplane-cse] adding namespace I0326 23:34:26.999613 4021 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.042825ms I0326 23:34:26.999625 4021 namespace.go:98] [openshift-console-user-settings] adding namespace I0326 23:34:27.000043 4021 namespace.go:102] [kube-public] adding namespace took 8.450156ms I0326 23:34:27.000060 4021 namespace.go:98] [openshift-backplane-srep] adding namespace I0326 23:34:27.000435 4021 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.26385ms I0326 23:34:27.000453 4021 namespace.go:98] [openshift-config] adding namespace I0326 23:34:27.000847 4021 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.070925ms I0326 23:34:27.000860 4021 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0326 23:34:27.001271 4021 namespace.go:102] [openshift-dns] adding namespace took 6.891146ms I0326 23:34:27.001286 4021 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0326 23:34:27.001675 4021 namespace.go:102] [openshift-package-operator] adding namespace took 6.721308ms I0326 23:34:27.001692 4021 namespace.go:98] [openshift-apiserver] adding namespace I0326 23:34:27.002059 4021 namespace.go:102] [openshift] adding namespace took 6.577765ms I0326 23:34:27.002074 4021 namespace.go:98] [openshift-network-console] adding namespace I0326 23:34:27.002445 4021 namespace.go:102] [openshift-logging] adding namespace took 6.465344ms I0326 23:34:27.002459 4021 namespace.go:98] [openshift-backplane] adding namespace I0326 23:34:27.002899 4021 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.438111ms I0326 23:34:27.002915 4021 namespace.go:98] [openshift-host-network] adding namespace I0326 23:34:27.003301 4021 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.379684ms I0326 23:34:27.003316 4021 namespace.go:98] [openshift-config-operator] adding namespace I0326 23:34:27.003763 4021 namespace.go:102] [default] adding namespace took 6.391663ms I0326 23:34:27.003779 4021 namespace.go:98] [kube-node-lease] adding namespace I0326 23:34:27.004192 4021 namespace.go:102] [dedicated-admin] adding namespace took 6.374628ms I0326 23:34:27.004208 4021 namespace.go:98] [openshift-backplane-csm] adding namespace I0326 23:34:27.004623 4021 namespace.go:102] [openshift-ingress] adding namespace took 6.348619ms I0326 23:34:27.004638 4021 namespace.go:98] [openshift-kube-apiserver] adding namespace I0326 23:34:27.004994 4021 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.260236ms I0326 23:34:27.005008 4021 namespace.go:98] [openshift-backplane-mobb] adding namespace I0326 23:34:27.005366 4021 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.197915ms I0326 23:34:27.005377 4021 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0326 23:34:27.005794 4021 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.163456ms I0326 23:34:27.005805 4021 namespace.go:98] [openshift-image-registry] adding namespace I0326 23:34:27.006172 4021 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.106381ms I0326 23:34:27.006185 4021 namespace.go:98] [openshift-operators] adding namespace I0326 23:34:27.006627 4021 namespace.go:102] [openshift-config] adding namespace took 6.16725ms I0326 23:34:27.007008 4021 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.14297ms I0326 23:34:27.007022 4021 namespace.go:98] [openshift-backplane-cee] adding namespace I0326 23:34:27.007415 4021 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.124128ms I0326 23:34:27.007430 4021 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0326 23:34:27.007799 4021 namespace.go:102] [openshift-apiserver] adding namespace took 6.101504ms I0326 23:34:27.007810 4021 namespace.go:98] [openshift-backplane-tam] adding namespace I0326 23:34:27.008196 4021 namespace.go:102] [openshift-network-console] adding namespace took 6.115741ms I0326 23:34:27.008207 4021 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0326 23:34:27.008578 4021 namespace.go:102] [openshift-backplane] adding namespace took 6.110571ms I0326 23:34:27.008596 4021 namespace.go:98] [openshift-machine-api] adding namespace I0326 23:34:27.009175 4021 namespace.go:102] [openshift-host-network] adding namespace took 6.253221ms I0326 23:34:27.009189 4021 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0326 23:34:27.009727 4021 namespace.go:102] [openshift-config-operator] adding namespace took 6.404914ms I0326 23:34:27.009740 4021 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0326 23:34:27.010280 4021 namespace.go:102] [kube-node-lease] adding namespace took 6.494232ms I0326 23:34:27.010294 4021 namespace.go:98] [openshift-ingress-operator] adding namespace I0326 23:34:27.010854 4021 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.640208ms I0326 23:34:27.010875 4021 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0326 23:34:27.011373 4021 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.728915ms I0326 23:34:27.011387 4021 namespace.go:98] [openshift-monitoring] adding namespace I0326 23:34:27.011860 4021 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.844401ms I0326 23:34:27.011876 4021 namespace.go:98] [openshift-node] adding namespace I0326 23:34:27.012285 4021 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.902184ms I0326 23:34:27.012301 4021 namespace.go:98] [openshift-config-managed] adding namespace I0326 23:34:27.012861 4021 namespace.go:102] [openshift-image-registry] adding namespace took 7.050767ms I0326 23:34:27.012875 4021 namespace.go:98] [kube-system] adding namespace I0326 23:34:27.013258 4021 namespace.go:102] [openshift-operators] adding namespace took 7.068585ms I0326 23:34:27.013274 4021 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0326 23:34:27.013684 4021 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.656795ms I0326 23:34:27.013699 4021 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0326 23:34:27.014084 4021 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.65009ms I0326 23:34:27.014099 4021 namespace.go:98] [openshift-operators-redhat] adding namespace I0326 23:34:27.014422 4021 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.608135ms I0326 23:34:27.014435 4021 namespace.go:98] [openshift-cluster-version] adding namespace I0326 23:34:27.015301 4021 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.089298ms I0326 23:34:27.015314 4021 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0326 23:34:27.015803 4021 namespace.go:102] [openshift-machine-api] adding namespace took 7.201859ms I0326 23:34:27.015815 4021 namespace.go:98] [openshift-etcd] adding namespace I0326 23:34:27.016180 4021 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.985782ms I0326 23:34:27.016194 4021 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0326 23:34:27.016693 4021 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.949006ms I0326 23:34:27.016706 4021 namespace.go:98] [open-cluster-management-2p9n1rl9ff1n78gavi22o5cr9k7iopuv] adding namespace I0326 23:34:27.017085 4021 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.785391ms I0326 23:34:27.017097 4021 namespace.go:98] [openshift-ingress-canary] adding namespace I0326 23:34:27.017485 4021 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.604335ms I0326 23:34:27.017497 4021 namespace.go:98] [openshift-console-operator] adding namespace I0326 23:34:27.017864 4021 namespace.go:102] [openshift-monitoring] adding namespace took 6.471732ms I0326 23:34:27.017874 4021 namespace.go:98] [openshift-authentication] adding namespace I0326 23:34:27.018238 4021 namespace.go:102] [openshift-node] adding namespace took 6.357656ms I0326 23:34:27.018250 4021 namespace.go:98] [openshift-customer-monitoring] adding namespace W0326 23:34:27.018599 4021 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-fxv4k: pod kube-system/global-pull-secret-syncer-fxv4k: no pod IPs found I0326 23:34:27.018644 4021 namespace.go:102] [openshift-config-managed] adding namespace took 6.337651ms I0326 23:34:27.018653 4021 namespace.go:98] [openshift-network-operator] adding namespace I0326 23:34:27.019058 4021 namespace.go:102] [kube-system] adding namespace took 6.177686ms I0326 23:34:27.019068 4021 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0326 23:34:27.019462 4021 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.180274ms I0326 23:34:27.019478 4021 namespace.go:98] [openshift-network-node-identity] adding namespace I0326 23:34:27.020185 4021 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.480786ms I0326 23:34:27.020581 4021 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.475959ms I0326 23:34:27.021017 4021 namespace.go:102] [openshift-cluster-version] adding namespace took 6.575358ms I0326 23:34:27.021424 4021 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.105289ms I0326 23:34:27.021832 4021 namespace.go:102] [openshift-etcd] adding namespace took 6.012387ms I0326 23:34:27.022200 4021 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.00203ms I0326 23:34:27.022618 4021 namespace.go:102] [open-cluster-management-2p9n1rl9ff1n78gavi22o5cr9k7iopuv] adding namespace took 5.908056ms I0326 23:34:27.022976 4021 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.874505ms I0326 23:34:27.023363 4021 namespace.go:102] [openshift-console-operator] adding namespace took 5.861649ms I0326 23:34:27.023374 4021 namespace.go:98] [openshift-infra] adding namespace I0326 23:34:27.023794 4021 namespace.go:102] [openshift-authentication] adding namespace took 5.914611ms I0326 23:34:27.024189 4021 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.934919ms I0326 23:34:27.024201 4021 namespace.go:98] [openshift-multus] adding namespace I0326 23:34:27.024594 4021 namespace.go:102] [openshift-network-operator] adding namespace took 5.937129ms I0326 23:34:27.024607 4021 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0326 23:34:27.025024 4021 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.95204ms I0326 23:34:27.025035 4021 namespace.go:98] [openshift-must-gather-operator] adding namespace I0326 23:34:27.025419 4021 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.935183ms W0326 23:34:27.025781 4021 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-twhht: pod openshift-multus/network-metrics-daemon-twhht: no pod IPs found W0326 23:34:27.025792 4021 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-vbqq7: pod openshift-multus/network-metrics-daemon-vbqq7: no pod IPs found I0326 23:34:27.025804 4021 namespace.go:102] [openshift-infra] adding namespace took 2.42517ms I0326 23:34:27.026217 4021 namespace.go:102] [openshift-multus] adding namespace took 2.011502ms I0326 23:34:27.026616 4021 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 2.003636ms I0326 23:34:27.026943 4021 namespace.go:102] [openshift-must-gather-operator] adding namespace took 1.904264ms I0326 23:34:27.027288 4021 zone_ic_handler.go:186] Time taken to create transit switch: 253.277µs I0326 23:34:27.027375 4021 master.go:532] Adding or Updating local node "ip-10-0-2-65.ec2.internal" for network "default" I0326 23:34:27.027450 4021 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-85.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-85.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-85.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-85.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-85.ec2.internal I0326 23:34:27.027942 4021 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-245.ec2.internal for the network default I0326 23:34:27.028865 4021 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-245.ec2.internal" on network "default" took: 934.036µs I0326 23:34:27.030798 4021 default_network_controller.go:806] Node add failed for ip-10-0-2-65.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-65.ec2.internal, error getting gateway config for node ip-10-0-2-65.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-65.ec2.internal", failed to update chassis to local for local node ip-10-0-2-65.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-65.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-65.ec2.internal] E0326 23:34:27.030857 4021 obj_retry.go:541] Failed to create *v1.Node ip-10-0-2-65.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-65.ec2.internal, error getting gateway config for node ip-10-0-2-65.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-65.ec2.internal", failed to update chassis to local for local node ip-10-0-2-65.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-65.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-65.ec2.internal] I0326 23:34:27.030905 4021 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0326 23:34:27.030930 4021 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0326 23:34:27.030944 4021 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0326 23:34:27.030965 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0326 23:34:27.030978 4021 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-65.ec2.internal", UID:"c0a7c519-2b5d-45f1-8d81-afd4e64d1db1", APIVersion:"v1", ResourceVersion:"8894", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-65.ec2.internal, error getting gateway config for node ip-10-0-2-65.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-65.ec2.internal", failed to update chassis to local for local node ip-10-0-2-65.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-65.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-65.ec2.internal] I0326 23:34:27.030990 4021 node_tracker.go:235] Node ip-10-0-0-85.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-85.ec2.internal" I0326 23:34:27.031006 4021 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:27.031015 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:27.031021 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-245.ec2.internal in network "default" I0326 23:34:27.031062 4021 node_tracker.go:169] Node ip-10-0-1-245.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:27.031069 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:27.031075 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-65.ec2.internal in network "default" I0326 23:34:27.031084 4021 node_tracker.go:235] Node ip-10-0-2-65.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-65.ec2.internal" I0326 23:34:27.031091 4021 node_tracker.go:169] Node ip-10-0-2-65.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:27.031095 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:27.064411 4021 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:34:27.070464 4021 iptables.go:108] Creating table: filter chain: OUTPUT I0326 23:34:27.076463 4021 gateway_shared_intf.go:1787] Gateway Creation Complete I0326 23:34:27.076480 4021 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-65.ec2.internal","mac-address":"0e:5c:83:43:e9:2f","ip-addresses":["10.0.2.65/24"],"ip-address":"10.0.2.65/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:1ee003de-9bfd-4c80-8f11-39143ee68959 k8s.ovn.org/node-encap-ips:["10.0.2.65"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.65/24"} k8s.ovn.org/zone-name:ip-10-0-2-65.ec2.internal] on node ip-10-0-2-65.ec2.internal I0326 23:34:27.090923 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-65.ec2.internal in network "default" I0326 23:34:27.090940 4021 master.go:532] Adding or Updating local node "ip-10-0-2-65.ec2.internal" for network "default" I0326 23:34:27.090992 4021 node_tracker.go:169] Node ip-10-0-2-65.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:27.091012 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:27.094182 4021 default_node_network_controller.go:518] Node connection status = connected W0326 23:34:27.096252 4021 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-twhht: pod openshift-multus/network-metrics-daemon-twhht: no pod IPs found W0326 23:34:27.096269 4021 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-xm5zg: pod openshift-network-diagnostics/network-check-target-xm5zg: no pod IPs found W0326 23:34:27.096277 4021 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-fxv4k: pod kube-system/global-pull-secret-syncer-fxv4k: no pod IPs found I0326 23:34:27.097222 4021 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-65.ec2.internal for the network default I0326 23:34:27.131817 4021 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0326 23:34:27.131842 4021 services_controller.go:208] Setting up event handlers for services for network=default I0326 23:34:27.131986 4021 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0326 23:34:27.132050 4021 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0326 23:34:27.132070 4021 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0326 23:34:27.232320 4021 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0326 23:34:27.232361 4021 repair.go:60] Starting repairing loop for services I0326 23:34:27.232456 4021 repair.go:131] Deleted 0 stale service LBs I0326 23:34:27.232477 4021 repair.go:137] Deleted 0 stale Chassis Template Vars I0326 23:34:27.232495 4021 repair.go:62] Finished repairing loop for services: 152.97µs I0326 23:34:27.232530 4021 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0326 23:34:27.232540 4021 services_controller.go:254] Starting workers for network=default I0326 23:34:27.232917 4021 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-6k2z4 of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0326 23:34:27.233025 4021 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-vbqq7, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-vbqq7: suppressed error logged: pod openshift-multus/network-metrics-daemon-vbqq7: no pod IPs found I0326 23:34:27.233025 4021 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-rz6dg, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rz6dg: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rz6dg: no pod IPs found I0326 23:34:27.233087 4021 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-twhht] creating logical port openshift-multus_network-metrics-daemon-twhht for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:27.233167 4021 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-fxv4k] creating logical port kube-system_global-pull-secret-syncer-fxv4k for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:27.233090 4021 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xm5zg] creating logical port openshift-network-diagnostics_network-check-target-xm5zg for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:27.233230 4021 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-twhht I0326 23:34:27.233251 4021 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-fxv4k I0326 23:34:27.233319 4021 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-xm5zg I0326 23:34:27.249618 4021 pod.go:62] [openshift-multus/network-metrics-daemon-twhht] pod update took 16.454ms I0326 23:34:27.249639 4021 pod.go:62] [openshift-network-diagnostics/network-check-target-xm5zg] pod update took 16.358945ms I0326 23:34:27.249653 4021 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-xm5zg] addLogicalPort annotation time took 16.374892ms I0326 23:34:27.249641 4021 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-twhht] addLogicalPort annotation time took 16.482428ms I0326 23:34:27.250809 4021 pods.go:271] [openshift-network-diagnostics/network-check-target-xm5zg] addLogicalPort took 17.712402ms, libovsdb time 897.01µs I0326 23:34:27.250965 4021 pods.go:271] [openshift-multus/network-metrics-daemon-twhht] addLogicalPort took 17.88506ms, libovsdb time 1.067545ms I0326 23:34:27.252340 4021 pod.go:62] [kube-system/global-pull-secret-syncer-fxv4k] pod update took 19.12667ms I0326 23:34:27.252367 4021 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-fxv4k] addLogicalPort annotation time took 19.158825ms I0326 23:34:27.253095 4021 pods.go:271] [kube-system/global-pull-secret-syncer-fxv4k] addLogicalPort took 19.935037ms, libovsdb time 552.347µs I0326 23:34:27.253929 4021 admin_network_policy_controller.go:227] Starting controller default-network-controller I0326 23:34:27.253948 4021 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0326 23:34:27.253961 4021 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0326 23:34:27.253965 4021 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0326 23:34:27.254138 4021 repair.go:29] Repairing admin network policies took 168.776µs I0326 23:34:27.254235 4021 repair.go:92] Repairing baseline admin network policies took 89.108µs I0326 23:34:27.254617 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0326 23:34:27.254638 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0326 23:34:27.256104 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0326 23:34:27.256125 4021 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0326 23:34:27.256137 4021 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0326 23:34:27.258263 4021 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0326 23:34:27.258281 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0326 23:34:27.258295 4021 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0326 23:34:27.259731 4021 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0326 23:34:27.259748 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0326 23:34:27.259755 4021 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0326 23:34:27.260026 4021 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0326 23:34:27.260044 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0326 23:34:27.260746 4021 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0326 23:34:27.260759 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:34:27.262383 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0326 23:34:27.262400 4021 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0326 23:34:27.262408 4021 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0326 23:34:27.263881 4021 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0326 23:34:27.263896 4021 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0326 23:34:27.263906 4021 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0326 23:34:27.265058 4021 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0326 23:34:27.265077 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0326 23:34:27.265087 4021 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0326 23:34:27.265838 4021 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0326 23:34:27.265858 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0326 23:34:27.265867 4021 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0326 23:34:27.266356 4021 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0326 23:34:27.266374 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0326 23:34:27.266381 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0326 23:34:27.266640 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0326 23:34:27.266655 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0326 23:34:27.266662 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0326 23:34:27.267862 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0326 23:34:27.267878 4021 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0326 23:34:27.267887 4021 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0326 23:34:27.269131 4021 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0326 23:34:27.269150 4021 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0326 23:34:27.269159 4021 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0326 23:34:27.271004 4021 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0326 23:34:27.271020 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0326 23:34:27.271028 4021 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0326 23:34:27.271497 4021 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0326 23:34:27.271515 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0326 23:34:27.271532 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0326 23:34:27.272007 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:34:27.272025 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0326 23:34:27.272033 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0326 23:34:27.272519 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:34:27.272536 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0326 23:34:27.272543 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0326 23:34:27.273025 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0326 23:34:27.273040 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0326 23:34:27.273049 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0326 23:34:27.273523 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0326 23:34:27.273539 4021 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0326 23:34:27.273549 4021 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0326 23:34:27.274481 4021 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0326 23:34:27.274497 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0326 23:34:27.274505 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0326 23:34:27.274945 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0326 23:34:27.274962 4021 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0326 23:34:27.274971 4021 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0326 23:34:27.275239 4021 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0326 23:34:27.275255 4021 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0326 23:34:27.275261 4021 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0326 23:34:27.275537 4021 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0326 23:34:27.275551 4021 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0326 23:34:27.275559 4021 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0326 23:34:27.277118 4021 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0326 23:34:27.277133 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0326 23:34:27.277140 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0326 23:34:27.277656 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0326 23:34:27.277676 4021 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0326 23:34:27.277684 4021 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0326 23:34:27.278204 4021 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0326 23:34:27.278219 4021 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0326 23:34:27.278243 4021 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:34:27.278892 4021 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0326 23:34:27.278912 4021 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0326 23:34:27.278920 4021 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0326 23:34:27.279916 4021 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0326 23:34:27.279935 4021 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0326 23:34:27.279944 4021 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0326 23:34:27.280804 4021 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0326 23:34:27.280823 4021 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0326 23:34:27.280836 4021 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0326 23:34:27.281905 4021 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0326 23:34:27.282143 4021 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-6k2z4 of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0326 23:34:27.287728 4021 egressfirewall.go:319] Starting EgressFirewall controller I0326 23:34:27.287752 4021 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0326 23:34:27.287772 4021 controller.go:133] Adding controller egress-firewall-controller event handlers I0326 23:34:27.287781 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0326 23:34:27.287791 4021 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0326 23:34:27.287794 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0326 23:34:27.287804 4021 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0326 23:34:27.288084 4021 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0326 23:34:27.288096 4021 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0326 23:34:27.288107 4021 egressqos.go:193] Setting up event handlers for EgressQoS I0326 23:34:27.288187 4021 egressfirewall.go:628] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0326 23:34:27.288202 4021 egressfirewall.go:628] Syncing node "ip-10-0-1-245.ec2.internal" for egress firewall I0326 23:34:27.288208 4021 egressfirewall.go:628] Syncing node "ip-10-0-2-65.ec2.internal" for egress firewall I0326 23:34:27.288338 4021 egressqos.go:245] Starting EgressQoS Controller I0326 23:34:27.288366 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0326 23:34:27.288388 4021 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0326 23:34:27.288395 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0326 23:34:27.288400 4021 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0326 23:34:27.288405 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0326 23:34:27.288409 4021 shared_informer.go:356] "Caches are synced" controller="egressqos" I0326 23:34:27.288412 4021 egressqos.go:259] Repairing EgressQoSes I0326 23:34:27.288416 4021 egressqos.go:399] Starting repairing loop for egressqos I0326 23:34:27.288485 4021 egressqos.go:401] Finished repairing loop for egressqos: 68.861µs I0326 23:34:27.288504 4021 egressservice_zone.go:132] Setting up event handlers for Egress Services I0326 23:34:27.288531 4021 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-65.ec2.internal I0326 23:34:27.288603 4021 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-65.ec2.internal : 74.155µs I0326 23:34:27.288797 4021 egressservice_zone.go:210] Starting Egress Services Controller I0326 23:34:27.288811 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:34:27.288821 4021 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:34:27.288829 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:34:27.288836 4021 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:34:27.288849 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:34:27.288857 4021 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:34:27.288865 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0326 23:34:27.288872 4021 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0326 23:34:27.288876 4021 egressservice_zone.go:228] Repairing Egress Services I0326 23:34:27.289163 4021 master_controller.go:88] Starting Admin Policy Based Route Controller I0326 23:34:27.289173 4021 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:34:27.289232 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0326 23:34:27.289241 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 10.837µs I0326 23:34:27.289248 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-245.ec2.internal I0326 23:34:27.289259 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-245.ec2.internal: 11.608µs I0326 23:34:27.289271 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-65.ec2.internal I0326 23:34:27.289275 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-65.ec2.internal: 4.316µs I0326 23:34:27.289280 4021 default_network_controller.go:568] Completing all the Watchers took 297.98683ms I0326 23:34:27.289296 4021 default_network_controller.go:572] Starting unidling controllers I0326 23:34:27.289317 4021 unidle.go:46] Registering OVN SB ControllerEvent handler I0326 23:34:27.289331 4021 unidle.go:63] Populating Initial ContollerEvent events I0326 23:34:27.289361 4021 unidle.go:79] Setting up event handlers for services I0326 23:34:27.289423 4021 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:34:27.594707 4021 default_node_network_controller.go:518] Node connection status = connected I0326 23:34:28.094568 4021 default_node_network_controller.go:518] Node connection status = connected I0326 23:34:28.545661 4021 default_network_controller.go:1008] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:34:28.594854 4021 default_node_network_controller.go:518] Node connection status = connected I0326 23:34:29.094561 4021 default_node_network_controller.go:518] Node connection status = connected I0326 23:34:29.339784 4021 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:34:29.594204 4021 default_node_network_controller.go:518] Node connection status = connected I0326 23:34:29.609379 4021 bridgeconfig.go:537] Gateway is ready I0326 23:34:29.609398 4021 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0326 23:34:29.619958 4021 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.65. Updating OVN encap IP. I0326 23:34:29.624670 4021 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.65 - it is already configured I0326 23:34:29.624684 4021 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.65/24:{} 172.20.0.1/32:{}]. Updating annotations. I0326 23:34:29.625012 4021 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.65/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-65.ec2.internal","mac-address":"0e:5c:83:43:e9:2f","ip-addresses":["10.0.2.65/24"],"ip-address":"10.0.2.65/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:1ee003de-9bfd-4c80-8f11-39143ee68959 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.65/24"}] on node ip-10-0-2-65.ec2.internal I0326 23:34:29.639531 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-65.ec2.internal in network "default" I0326 23:34:29.639559 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-65.ec2.internal I0326 23:34:29.639573 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-65.ec2.internal: 17.504µs I0326 23:34:29.639586 4021 node_tracker.go:169] Node ip-10-0-2-65.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:29.639595 4021 egressfirewall.go:628] Syncing node "ip-10-0-2-65.ec2.internal" for egress firewall I0326 23:34:29.639603 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:29.639665 4021 master.go:532] Adding or Updating local node "ip-10-0-2-65.ec2.internal" for network "default" I0326 23:34:29.640436 4021 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.65 I0326 23:34:29.640681 4021 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-65.ec2.internal I0326 23:34:29.644382 4021 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-65.ec2.internal. Updating no re-route policies I0326 23:34:29.644392 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xm5zg I0326 23:34:29.644404 4021 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-n8czt I0326 23:34:29.644420 4021 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-n8czt I0326 23:34:29.644421 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-pv2c5 I0326 23:34:29.644430 4021 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-fxv4k I0326 23:34:29.644438 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-pv2c5 I0326 23:34:29.644438 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-vbqq7 I0326 23:34:29.644443 4021 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-fxv4k I0326 23:34:29.644392 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-twhht I0326 23:34:29.644451 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-vbqq7 I0326 23:34:29.644452 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-wjl7g I0326 23:34:29.644454 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-twhht I0326 23:34:29.644459 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-wjl7g I0326 23:34:29.644462 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-pv2c5 after 0 failed attempt(s) I0326 23:34:29.644421 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rz6dg I0326 23:34:29.644468 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-dhfnk I0326 23:34:29.644478 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rz6dg I0326 23:34:29.644484 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-gzw7m I0326 23:34:29.644486 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-dhfnk I0326 23:34:29.644445 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xm5zg I0326 23:34:29.644491 4021 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-twhht] creating logical port openshift-multus_network-metrics-daemon-twhht for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:29.644443 4021 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-n8czt after 0 failed attempt(s) I0326 23:34:29.644494 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-gzw7m I0326 23:34:29.644499 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-65.ec2.internal I0326 23:34:29.644507 4021 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-65.ec2.internal I0326 23:34:29.644486 4021 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-fxv4k] creating logical port kube-system_global-pull-secret-syncer-fxv4k for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:29.644514 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-65.ec2.internal I0326 23:34:29.644515 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-w4swb I0326 23:34:29.644520 4021 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-65.ec2.internal I0326 23:34:29.644525 4021 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xm5zg] creating logical port openshift-network-diagnostics_network-check-target-xm5zg for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:29.644535 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-w4swb I0326 23:34:29.644541 4021 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-65.ec2.internal after 0 failed attempt(s) I0326 23:34:29.644486 4021 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-vbqq7, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-vbqq7: suppressed error logged: pod openshift-multus/network-metrics-daemon-vbqq7: no pod IPs found I0326 23:34:29.644516 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-dhfnk after 0 failed attempt(s) I0326 23:34:29.644550 4021 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-rz6dg, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rz6dg: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rz6dg: no pod IPs found I0326 23:34:29.644478 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-wjl7g after 0 failed attempt(s) I0326 23:34:29.644570 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-w4swb after 0 failed attempt(s) I0326 23:34:29.644405 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-l6r9w I0326 23:34:29.644588 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-l6r9w I0326 23:34:29.644514 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-gzw7m after 0 failed attempt(s) I0326 23:34:29.644602 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-l6r9w after 0 failed attempt(s) I0326 23:34:29.644614 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-65.ec2.internal after 0 failed attempt(s) I0326 23:34:29.644450 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-ltk6s I0326 23:34:29.644485 4021 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-mbqdg I0326 23:34:29.644637 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-ltk6s I0326 23:34:29.644652 4021 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-mbqdg I0326 23:34:29.644678 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-mbqdg after 0 failed attempt(s) I0326 23:34:29.644684 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-ltk6s after 0 failed attempt(s) I0326 23:34:29.645462 4021 pods.go:271] [openshift-network-diagnostics/network-check-target-xm5zg] addLogicalPort took 949.529µs, libovsdb time 336.008µs I0326 23:34:29.645474 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-xm5zg after 0 failed attempt(s) I0326 23:34:29.645580 4021 pods.go:271] [openshift-multus/network-metrics-daemon-twhht] addLogicalPort took 1.100112ms, libovsdb time 396.356µs I0326 23:34:29.645585 4021 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-twhht after 0 failed attempt(s) I0326 23:34:29.645652 4021 pods.go:271] [kube-system/global-pull-secret-syncer-fxv4k] addLogicalPort took 1.183954ms, libovsdb time 466.572µs I0326 23:34:29.645674 4021 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-fxv4k after 0 failed attempt(s) I0326 23:34:29.654944 4021 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0326 23:34:29.665786 4021 iptables.go:108] Creating table: mangle chain: OUTPUT I0326 23:34:29.669001 4021 iptables.go:108] Creating table: nat chain: OUTPUT I0326 23:34:29.699736 4021 iptables.go:108] Creating table: nat chain: PREROUTING I0326 23:34:29.770022 4021 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:34:29.786589 4021 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0326 23:34:29.786718 4021 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0326 23:34:29.786782 4021 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0326 23:34:29.786864 4021 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0326 23:34:29.786955 4021 gateway_init.go:325] Gateway and management port readiness took 2.696218835s I0326 23:34:29.786981 4021 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0326 23:34:29.787038 4021 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0326 23:34:29.787052 4021 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0326 23:34:29.787060 4021 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:34:29.787122 4021 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0326 23:34:29.787202 4021 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.55µs I0326 23:34:29.787502 4021 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0326 23:34:29.787534 4021 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 22.509µs I0326 23:34:29.787543 4021 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 41.365µs I0326 23:34:29.787548 4021 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0326 23:34:29.787557 4021 nad_controller.go:173] [node-nad-controller NAD controller]: started I0326 23:34:29.787561 4021 default_node_network_controller.go:898] Starting the default node network controller I0326 23:34:29.787565 4021 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0326 23:34:29.787586 4021 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.484µs I0326 23:34:29.787725 4021 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0326 23:34:29.787733 4021 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0326 23:34:29.787755 4021 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b80950 0xc00033ec40 0xc00033ed20 0xc00033ed90 0xc00033ee70 0xc00033ef50 0xc00033efc0 0xc000bcc780 0xc00033f0a0 0xc00033f110 0xc000148280 0xc0062befc0 0xc00033f180 map[0x2f67040:0xc003092540 0x2f67580:0xc0030926c0 0x2f68000:0xc002598cc0 0x2f682a0:0xc0030924e0 0x2f687e0:0xc0030925a0 0x2f68a80:0xc003092600 0x2f68d20:0xc003092660 0x2f69500:0xc0030928a0 0x2f697a0:0xc003092900 0x2f93d40:0xc001116fc0 0x2f940e0:0xc001117020 0x2f94bc0:0xc000b935c0 0x2f96520:0xc000c818c0 0x2f97000:0xc001117140 0x2f97e80:0xc000c81920] 0xc00016ccb0 0} I0326 23:34:29.787812 4021 gateway.go:151] Starting gateway service sync I0326 23:34:29.788264 4021 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0326 23:34:29.790358 4021 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0326 23:34:29.817575 4021 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0326 23:34:29.818791 4021 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0326 23:34:29.819954 4021 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0326 23:34:29.854794 4021 gateway.go:168] Gateway service sync done. Time taken: 66.967741ms I0326 23:34:29.855185 4021 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31791 I0326 23:34:29.855592 4021 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b80950 0xc00033ec40 0xc00033ed20 0xc00033ed90 0xc00033ee70 0xc00033ef50 0xc00033efc0 0xc000bcc780 0xc00033f0a0 0xc00033f110 0xc000148280 0xc0062befc0 0xc00033f180 map[0x2f67040:0xc003092540 0x2f67580:0xc0030926c0 0x2f68000:0xc002598cc0 0x2f682a0:0xc0030924e0 0x2f687e0:0xc0030925a0 0x2f68a80:0xc003092600 0x2f68d20:0xc003092660 0x2f69500:0xc0030928a0 0x2f697a0:0xc003092900 0x2f93d40:0xc001116fc0 0x2f940e0:0xc001117020 0x2f94bc0:0xc000b935c0 0x2f96520:0xc000c818c0 0x2f97000:0xc001117140 0x2f97e80:0xc000c81920] 0xc00016ccb0 0} I0326 23:34:29.856081 4021 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b80950 0xc00033ec40 0xc00033ed20 0xc00033ed90 0xc00033ee70 0xc00033ef50 0xc00033efc0 0xc000bcc780 0xc00033f0a0 0xc00033f110 0xc000148280 0xc0062befc0 0xc00033f180 map[0x2f67040:0xc003092540 0x2f67580:0xc0030926c0 0x2f68000:0xc002598cc0 0x2f682a0:0xc0030924e0 0x2f687e0:0xc0030925a0 0x2f68a80:0xc003092600 0x2f68d20:0xc003092660 0x2f69500:0xc0030928a0 0x2f697a0:0xc003092900 0x2f93d40:0xc001116fc0 0x2f940e0:0xc001117020 0x2f94bc0:0xc000b935c0 0x2f96520:0xc000c818c0 0x2f97000:0xc001117140 0x2f97e80:0xc000c81920] 0xc00016ccb0 0} I0326 23:34:29.856117 4021 gateway.go:519] Reconciling gateway with updates I0326 23:34:29.861893 4021 gateway.go:326] Spawning Conntrack Rule Check Thread I0326 23:34:29.861974 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:34:29.861988 4021 default_node_network_controller.go:943] Gateway and management port readiness took 74.246375ms I0326 23:34:29.861993 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:34:29.862003 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:34:29.862007 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:34:29.862019 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:34:29.862020 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:34:29.862022 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:34:29.862032 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:34:29.862044 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:34:29.862059 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:34:29.862067 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:34:29.862085 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:34:29.862084 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:34:29.862089 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:34:29.862093 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:34:29.862097 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:34:29.862099 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:34:29.861979 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:34:29.862104 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:34:29.862113 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:34:29.862114 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:34:29.861997 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:34:29.862125 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:34:29.862126 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:34:29.862130 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:34:29.862133 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:34:29.862132 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:34:29.862144 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:34:29.862150 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:34:29.862157 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:34:29.862158 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:34:29.862098 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:34:29.862177 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:34:29.862185 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:34:29.862178 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:34:29.862196 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:34:29.862194 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:34:29.862204 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:34:29.862209 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:34:29.862209 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:34:29.862217 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:34:29.862008 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:34:29.862225 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:34:29.862225 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:34:29.862223 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:34:29.862238 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:34:29.862250 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:34:29.862254 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:34:29.862257 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:34:29.862264 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:34:29.862210 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:34:29.862271 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:34:29.862281 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:34:29.862178 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:34:29.862193 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:34:29.862300 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:34:29.862299 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:34:29.862283 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:34:29.862312 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:34:29.862311 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:34:29.862315 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:34:29.862320 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:34:29.862323 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:34:29.862285 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:34:29.862300 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:34:29.862141 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:34:29.862361 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:34:29.862375 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:34:29.862257 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:34:29.862387 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:34:29.862391 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:34:29.862391 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:34:29.862397 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:34:29.862395 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:34:29.862405 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:34:29.862413 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:34:29.862425 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:34:29.862389 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:34:29.862069 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:34:29.862479 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:34:29.862089 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:34:29.862238 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:34:29.862078 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:34:29.862091 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:34:29.862104 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:34:29.862498 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:34:29.862504 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:34:29.862102 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:34:29.862514 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:34:29.862158 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:34:29.862522 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:34:29.862534 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:34:29.862118 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:34:29.862158 4021 port_claim.go:71] Svc openshift-ingress/router-default: port 32268 is already open, no action needed I0326 23:34:29.862154 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:34:29.862577 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:34:29.862578 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:34:29.862117 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:34:29.862591 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:34:29.862121 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:34:29.862177 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:34:29.862262 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:34:29.862686 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:34:29.862266 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:34:29.862269 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:34:29.862283 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:34:29.862738 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:34:29.862292 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:34:29.862307 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:34:29.862378 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:34:29.862395 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:34:29.862793 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:34:29.862800 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:34:29.862579 4021 port_claim.go:71] Svc openshift-ingress/router-default: port 30406 is already open, no action needed I0326 23:34:29.862153 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:34:29.862867 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:34:29.862598 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:34:29.862192 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:34:29.863170 4021 node_ip_handler_linux.go:160] Node IP manager is running W0326 23:34:29.899140 4021 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:34:29.899306 4021 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:34:29.899428 4021 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0326 23:34:29.908316 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:34:29.908366 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:34:29.908401 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:34:29.908426 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:34:29.908439 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:34:29.908453 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:34:29.908460 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:34:29.908467 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:34:29.908474 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:34:29.908480 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:34:29.908486 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:34:29.908491 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:34:29.908499 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:34:29.946172 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:34:29.946182 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:34:29.983717 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:34:31.551140 4021 default_node_network_controller.go:1364] Starting node controller node sync W0326 23:34:31.551160 4021 egressip_healthcheck.go:76] Health checking using insecure connection I0326 23:34:31.551239 4021 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0326 23:34:31.562619 4021 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.456264ms I0326 23:34:31.562748 4021 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-245.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:34:31.562787 4021 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-245.ec2.internal", IP: 10.0.1.245 to PMTUD blocking rules I0326 23:34:31.568757 4021 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0326 23:34:31.568954 4021 egressservice_node.go:93] Setting up event handlers for Egress Services I0326 23:34:31.569060 4021 egressservice_node.go:183] Starting Egress Services Controller I0326 23:34:31.569075 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:34:31.569085 4021 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:34:31.569092 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:34:31.569097 4021 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:34:31.569102 4021 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:34:31.569109 4021 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:34:31.569113 4021 egressservice_node.go:197] Repairing Egress Services I0326 23:34:31.578788 4021 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0326 23:34:31.578802 4021 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:34:31.578982 4021 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0326 23:34:31.579000 4021 link_network_manager.go:117] Link manager is running I0326 23:34:31.579017 4021 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0326 23:34:31.579063 4021 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0326 23:34:31.579621 4021 vrf_manager.go:128] VRF manager is running I0326 23:34:31.579901 4021 node_controller_manager.go:437] Removing flows to drop GARP I0326 23:34:31.579911 4021 gateway.go:519] Reconciling gateway with updates I0326 23:34:31.600300 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:34:31.600310 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:34:31.600322 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:34:31.600328 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:34:31.600339 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:34:31.600337 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:34:31.600357 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:34:31.600368 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:34:31.600389 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:34:31.600404 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:34:31.600413 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:34:31.600418 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:34:31.600424 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:34:31.600424 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600430 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:34:31.600431 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:34:31.600331 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:34:31.600432 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:34:31.600446 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:34:31.600451 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:34:31.600451 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:34:31.600455 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:34:31.600461 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:34:31.600464 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:34:31.600472 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:34:31.600480 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:34:31.600482 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600495 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:34:31.600504 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:34:31.600503 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600506 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:34:31.600511 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:34:31.600483 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600522 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:34:31.600523 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:34:31.600521 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:34:31.600535 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:34:31.600537 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:34:31.600540 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:34:31.600546 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:34:31.600541 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:34:31.600549 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:34:31.600551 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:34:31.600553 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:34:31.600564 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:34:31.600575 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:34:31.600583 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600590 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:34:31.600544 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:34:31.600611 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:34:31.600635 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:34:31.600569 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:34:31.600649 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:34:31.600657 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:34:31.600662 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:34:31.600690 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:34:31.600696 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:34:31.600697 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:34:31.600705 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:34:31.600706 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:34:31.600707 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:34:31.600714 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600717 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:34:31.600722 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:34:31.600723 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:34:31.600722 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:34:31.600724 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:34:31.600735 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:34:31.600736 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:34:31.600540 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:34:31.600743 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:34:31.600735 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:34:31.600756 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:34:31.600531 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:34:31.600785 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:34:31.600744 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:34:31.600804 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:34:31.600704 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:34:31.600822 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:34:31.600824 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600823 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:34:31.600862 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:34:31.600880 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:34:31.600729 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:34:31.600927 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600882 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:34:31.600553 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:34:31.600956 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:34:31.600525 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:34:31.600965 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:34:31.600552 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:34:31.601007 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:34:31.600555 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:34:31.601030 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:34:31.600558 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:34:31.600567 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:34:31.600701 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:34:31.601073 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:34:31.600706 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:34:31.600707 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:34:31.601098 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:34:31.600711 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:34:31.600706 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:34:31.601161 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:34:31.601166 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:34:31.600530 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:34:31.601177 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:34:31.600712 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:34:31.601214 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:34:31.600712 4021 port_claim.go:71] Svc openshift-ingress/router-default: port 32268 is already open, no action needed I0326 23:34:31.601255 4021 port_claim.go:71] Svc openshift-ingress/router-default: port 30406 is already open, no action needed I0326 23:34:31.600716 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:34:31.601269 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:34:31.601296 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:34:31.600725 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:34:31.601311 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:34:31.600727 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:34:31.601366 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:34:31.600760 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:34:31.600526 4021 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:34:31.601403 4021 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:34:31.638151 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:34:31.638182 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:34:31.638194 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:34:31.638207 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:34:31.638189 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:34:31.638228 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:34:31.638251 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:34:31.638259 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:34:31.638264 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:34:31.675755 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:34:31.675761 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:34:31.675779 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:34:31.713389 4021 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:34:33.467187 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-65.ec2.internal I0326 23:34:33.467215 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-65.ec2.internal: 34.463µs I0326 23:34:33.495764 4021 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6b5d664bf7-98zz6] creating logical port openshift-image-registry_image-registry-6b5d664bf7-98zz6 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:33.495918 4021 kube.go:256] Updating pod openshift-image-registry/image-registry-6b5d664bf7-98zz6 I0326 23:34:33.511493 4021 pod.go:62] [openshift-image-registry/image-registry-6b5d664bf7-98zz6] pod update took 15.605098ms I0326 23:34:33.511529 4021 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-6b5d664bf7-98zz6] addLogicalPort annotation time took 15.635958ms I0326 23:34:33.512756 4021 pods.go:271] [openshift-image-registry/image-registry-6b5d664bf7-98zz6] addLogicalPort took 17.009062ms, libovsdb time 939.617µs I0326 23:34:33.545072 4021 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-tkng8] creating logical port openshift-dns_dns-default-tkng8 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:33.545152 4021 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-t7fd6] creating logical port openshift-ingress-canary_ingress-canary-t7fd6 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:34:33.545200 4021 kube.go:256] Updating pod openshift-dns/dns-default-tkng8 I0326 23:34:33.545251 4021 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-t7fd6 I0326 23:34:33.559589 4021 pod.go:62] [openshift-ingress-canary/ingress-canary-t7fd6] pod update took 14.356909ms I0326 23:34:33.559617 4021 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-t7fd6] addLogicalPort annotation time took 14.388614ms I0326 23:34:33.560854 4021 pods.go:271] [openshift-ingress-canary/ingress-canary-t7fd6] addLogicalPort took 15.709196ms, libovsdb time 870.092µs I0326 23:34:33.562807 4021 pod.go:62] [openshift-dns/dns-default-tkng8] pod update took 17.638803ms I0326 23:34:33.562829 4021 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-tkng8] addLogicalPort annotation time took 17.664714ms I0326 23:34:33.564150 4021 pods.go:271] [openshift-dns/dns-default-tkng8] addLogicalPort took 19.093103ms, libovsdb time 897.847µs I0326 23:34:33.907928 4021 cni.go:403] [openshift-ingress-canary/ingress-canary-t7fd6 d4e35dac1e5070c89a5131331144e4a01a1d333df27339e36ba69754db4f73ec network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-t7fd6 d4e35dac1e5070c89a5131331144e4a01a1d333df27339e36ba69754db4f73ec network default NAD default] I0326 23:34:33.924684 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-t7fd6, hostIfaceName: d4e35dac1e5070c, network: default, NAD default, SandboxID: "d4e35dac1e5070c89a5131331144e4a01a1d333df27339e36ba69754db4f73ec", PCI device ID: , UID: "42d29801-25d2-411c-864c-a283552d2386", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0326 23:34:33.979455 4021 cni.go:424] [openshift-ingress-canary/ingress-canary-t7fd6 d4e35dac1e5070c89a5131331144e4a01a1d333df27339e36ba69754db4f73ec network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-t7fd6 d4e35dac1e5070c89a5131331144e4a01a1d333df27339e36ba69754db4f73ec network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:00:32:ed:00:47\",\"name\":\"d4e35dac1e5070c\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/658a741b-eb72-444b-8411-f51c5b5043b9\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:34:34.839205 4021 cni.go:403] [kube-system/global-pull-secret-syncer-fxv4k 85506b9135702d5fb85e10aaa17701600ea1fdc6cbab348ea85178371a7d7981 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-fxv4k 85506b9135702d5fb85e10aaa17701600ea1fdc6cbab348ea85178371a7d7981 network default NAD default] I0326 23:34:34.848682 4021 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-fxv4k, hostIfaceName: 85506b9135702d5, network: default, NAD default, SandboxID: "85506b9135702d5fb85e10aaa17701600ea1fdc6cbab348ea85178371a7d7981", PCI device ID: , UID: "5260cf7c-9d9c-4dbc-a745-4da4543b719b", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0326 23:34:34.891653 4021 cni.go:424] [kube-system/global-pull-secret-syncer-fxv4k 85506b9135702d5fb85e10aaa17701600ea1fdc6cbab348ea85178371a7d7981 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-fxv4k 85506b9135702d5fb85e10aaa17701600ea1fdc6cbab348ea85178371a7d7981 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:39:bd:46:cf:77\",\"name\":\"85506b9135702d5\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ecaed8ab-89e6-439c-b657-e1f733771302\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:34:38.139225 4021 egressfirewall.go:628] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0326 23:34:38.139233 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0326 23:34:38.139265 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 33.988µs I0326 23:34:38.139337 4021 default_network_controller.go:1008] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:34:43.090925 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0326 23:34:43.090977 4021 node_tracker.go:235] Node ip-10-0-0-85.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-85.ec2.internal" I0326 23:34:43.090985 4021 default_network_controller.go:1008] Node "ip-10-0-0-85.ec2.internal" in remote zone "ip-10-0-0-85.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:34:43.090994 4021 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:43.091013 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:43.171497 4021 default_network_controller.go:1008] Node "ip-10-0-0-85.ec2.internal" in remote zone "ip-10-0-0-85.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:34:43.211464 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0326 23:34:43.211522 4021 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:43.211537 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:43.211470 4021 default_network_controller.go:1008] Node "ip-10-0-0-85.ec2.internal" in remote zone "ip-10-0-0-85.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:34:43.212201 4021 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-85.ec2.internal for the network default I0326 23:34:43.213440 4021 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-85.ec2.internal" on network "default" took: 1.252318ms I0326 23:34:45.291742 4021 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:45.292327 4021 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:45.292383 4021 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:34:45.292398 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0326 23:34:45.292416 4021 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:34:45.292424 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0326 23:34:45.764226 4021 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0326 23:34:45.764283 4021 egressfirewall.go:628] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0326 23:34:45.764302 4021 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0326 23:34:45.764324 4021 services_controller.go:574] Full service sync requested for network=default I0326 23:34:45.764232 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0326 23:34:45.764401 4021 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 10.0.0.85 to PMTUD blocking rules I0326 23:34:45.764418 4021 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:34:45.764402 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 173.289µs I0326 23:34:45.770505 4021 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-85.ec2.internal. Updating no re-route policies I0326 23:34:45.779736 4021 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 10.0.0.85 to PMTUD blocking rules I0326 23:34:45.779754 4021 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:34:46.318193 4021 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:46.318217 4021 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:46.318230 4021 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:46.318240 4021 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nv7t6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:47.803200 4021 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:34:47.812549 4021 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:34:48.551562 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0326 23:34:48.551594 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 38.479µs W0326 23:34:51.190691 4021 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:34:55.728889 4021 cni.go:403] [openshift-network-diagnostics/network-check-target-xm5zg 90fd3d0769bc7c1630b001e3169ef097d13b7360a514443467f22ca3e45e842f network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-xm5zg 90fd3d0769bc7c1630b001e3169ef097d13b7360a514443467f22ca3e45e842f network default NAD default] I0326 23:34:55.737390 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-xm5zg, hostIfaceName: 90fd3d0769bc7c1, network: default, NAD default, SandboxID: "90fd3d0769bc7c1630b001e3169ef097d13b7360a514443467f22ca3e45e842f", PCI device ID: , UID: "591bf067-bdba-4d7d-881e-a4f919739837", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0326 23:34:55.776986 4021 cni.go:424] [openshift-network-diagnostics/network-check-target-xm5zg 90fd3d0769bc7c1630b001e3169ef097d13b7360a514443467f22ca3e45e842f network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-xm5zg 90fd3d0769bc7c1630b001e3169ef097d13b7360a514443467f22ca3e45e842f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:fd:eb:2a:ab:c2\",\"name\":\"90fd3d0769bc7c1\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18c497af-1126-4c24-b68f-5d13c8c14cae\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:34:58.311339 4021 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:34:58.826167 4021 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:35:22.197648 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:35:22.197671 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:35:29.831276 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:35:57.548214 4021 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-cxsfg for pod on switch ip-10-0-2-65.ec2.internal I0326 23:35:57.548337 4021 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg I0326 23:35:57.556447 4021 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-6fxzg] creating logical port openshift-insights_insights-operator-5577457d66-6fxzg for pod on switch ip-10-0-2-65.ec2.internal I0326 23:35:57.556533 4021 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-6fxzg I0326 23:35:57.570882 4021 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg] pod update took 22.570809ms I0326 23:35:57.570905 4021 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg] addLogicalPort annotation time took 22.595594ms I0326 23:35:57.572082 4021 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg] addLogicalPort took 23.881001ms, libovsdb time 927.247µs I0326 23:35:57.575567 4021 pod.go:62] [openshift-insights/insights-operator-5577457d66-6fxzg] pod update took 19.057157ms I0326 23:35:57.575584 4021 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-6fxzg] addLogicalPort annotation time took 19.08011ms I0326 23:35:57.576711 4021 pods.go:271] [openshift-insights/insights-operator-5577457d66-6fxzg] addLogicalPort took 20.279011ms, libovsdb time 854.463µs I0326 23:35:57.641807 4021 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-lxlb5] creating logical port openshift-console-operator_console-operator-8777bd9bb-lxlb5 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:35:57.641890 4021 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-lxlb5 I0326 23:35:57.648713 4021 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-56b54dbfd-7rf85] creating logical port openshift-ingress_router-default-56b54dbfd-7rf85 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:35:57.648790 4021 kube.go:256] Updating pod openshift-ingress/router-default-56b54dbfd-7rf85 I0326 23:35:57.654651 4021 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-764b98bc4f-cmq8t] creating logical port openshift-ingress_router-default-764b98bc4f-cmq8t for pod on switch ip-10-0-2-65.ec2.internal I0326 23:35:57.654757 4021 kube.go:256] Updating pod openshift-ingress/router-default-764b98bc4f-cmq8t I0326 23:35:57.658313 4021 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-lxlb5] pod update took 16.441953ms I0326 23:35:57.658335 4021 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-lxlb5] addLogicalPort annotation time took 16.465731ms I0326 23:35:57.659490 4021 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-lxlb5] addLogicalPort took 17.702282ms, libovsdb time 810.809µs I0326 23:35:57.670087 4021 pod.go:62] [openshift-ingress/router-default-56b54dbfd-7rf85] pod update took 21.319637ms I0326 23:35:57.670110 4021 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-56b54dbfd-7rf85] addLogicalPort annotation time took 21.344101ms I0326 23:35:57.671128 4021 pods.go:271] [openshift-ingress/router-default-56b54dbfd-7rf85] addLogicalPort took 22.427647ms, libovsdb time 734.455µs I0326 23:35:57.677962 4021 pod.go:62] [openshift-ingress/router-default-764b98bc4f-cmq8t] pod update took 23.230584ms I0326 23:35:57.677986 4021 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-764b98bc4f-cmq8t] addLogicalPort annotation time took 23.256024ms I0326 23:35:57.678892 4021 pods.go:271] [openshift-ingress/router-default-764b98bc4f-cmq8t] addLogicalPort took 24.2518ms, libovsdb time 679.378µs I0326 23:35:57.913327 4021 cni.go:403] [openshift-insights/insights-operator-5577457d66-6fxzg b7c4cb6afb4f837d993c3ac2b56c88503e1c24e2d23da51fee48732aadba2f27 network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-6fxzg b7c4cb6afb4f837d993c3ac2b56c88503e1c24e2d23da51fee48732aadba2f27 network default NAD default] I0326 23:35:57.923704 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-6fxzg, hostIfaceName: b7c4cb6afb4f837, network: default, NAD default, SandboxID: "b7c4cb6afb4f837d993c3ac2b56c88503e1c24e2d23da51fee48732aadba2f27", PCI device ID: , UID: "f6c6cca5-4766-458c-ab08-59a4bdccad90", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0326 23:35:57.965886 4021 cni.go:424] [openshift-insights/insights-operator-5577457d66-6fxzg b7c4cb6afb4f837d993c3ac2b56c88503e1c24e2d23da51fee48732aadba2f27 network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-6fxzg b7c4cb6afb4f837d993c3ac2b56c88503e1c24e2d23da51fee48732aadba2f27 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:72:21:4e:56:8e\",\"name\":\"b7c4cb6afb4f837\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f6dac04-e996-46b3-a77d-adb208275049\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:35:58.295980 4021 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-lxlb5 c1215421a35613824021df15b2c3e7111d0306d2ddf7593fb7fa5dd5bebcbabe network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-lxlb5 c1215421a35613824021df15b2c3e7111d0306d2ddf7593fb7fa5dd5bebcbabe network default NAD default] I0326 23:35:58.306973 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-lxlb5, hostIfaceName: c1215421a356138, network: default, NAD default, SandboxID: "c1215421a35613824021df15b2c3e7111d0306d2ddf7593fb7fa5dd5bebcbabe", PCI device ID: , UID: "354b002a-7003-4ba9-8dff-dd87e0c39153", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0326 23:35:58.345775 4021 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-lxlb5 c1215421a35613824021df15b2c3e7111d0306d2ddf7593fb7fa5dd5bebcbabe network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-lxlb5 c1215421a35613824021df15b2c3e7111d0306d2ddf7593fb7fa5dd5bebcbabe network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:b9:ae:dd:31:86\",\"name\":\"c1215421a356138\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5fdb355-abab-4718-ab11-3356372952b6\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:00.972028 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:36:01.613209 4021 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0326 23:36:01.614185 4021 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 950.725µs I0326 23:36:01.639221 4021 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:36:02.631420 4021 namespace.go:98] [openshift-service-ca] adding namespace I0326 23:36:02.632304 4021 namespace.go:102] [openshift-service-ca] adding namespace took 862.327µs I0326 23:36:02.651872 4021 namespace.go:142] [openshift-service-ca] updating namespace W0326 23:36:11.393681 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:36:13.806437 4021 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg b9f95c2ebb27148e3f28e83efd0602db1c0954c1fe3c1f0a09d7cf7ce4f1c712 network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg b9f95c2ebb27148e3f28e83efd0602db1c0954c1fe3c1f0a09d7cf7ce4f1c712 network default NAD default] I0326 23:36:13.816438 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-cxsfg, hostIfaceName: b9f95c2ebb27148, network: default, NAD default, SandboxID: "b9f95c2ebb27148e3f28e83efd0602db1c0954c1fe3c1f0a09d7cf7ce4f1c712", PCI device ID: , UID: "264191bb-c1d1-4be2-8fec-e3ecf31f1bdb", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0326 23:36:13.860078 4021 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg b9f95c2ebb27148e3f28e83efd0602db1c0954c1fe3c1f0a09d7cf7ce4f1c712 network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-cxsfg b9f95c2ebb27148e3f28e83efd0602db1c0954c1fe3c1f0a09d7cf7ce4f1c712 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:d5:b6:d8:09:22\",\"name\":\"b9f95c2ebb27148\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/11728c75-d011-4bd2-a1b1-3f6d444a084a\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:29.250617 4021 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:29.253294 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0326 23:36:29.263859 4021 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0326 23:36:29.263874 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0326 23:36:29.263880 4021 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0326 23:36:29.405286 4021 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-mv2qq] creating logical port openshift-insights_insights-runtime-extractor-mv2qq for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:29.405397 4021 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-mv2qq I0326 23:36:29.422051 4021 pod.go:62] [openshift-insights/insights-runtime-extractor-mv2qq] pod update took 16.678662ms I0326 23:36:29.422075 4021 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-mv2qq] addLogicalPort annotation time took 16.707701ms I0326 23:36:29.423500 4021 pods.go:271] [openshift-insights/insights-runtime-extractor-mv2qq] addLogicalPort took 18.231551ms, libovsdb time 1.111708ms I0326 23:36:29.760670 4021 cni.go:403] [openshift-insights/insights-runtime-extractor-mv2qq a914112185ee40cee3e9a43fd221947f0c0cd0d3574d65da9e6537e72500c6f0 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-mv2qq a914112185ee40cee3e9a43fd221947f0c0cd0d3574d65da9e6537e72500c6f0 network default NAD default] I0326 23:36:29.774236 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-mv2qq, hostIfaceName: a914112185ee40c, network: default, NAD default, SandboxID: "a914112185ee40cee3e9a43fd221947f0c0cd0d3574d65da9e6537e72500c6f0", PCI device ID: , UID: "097573bd-6e47-4ff7-9b0e-9d7857318587", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0326 23:36:29.818699 4021 cni.go:424] [openshift-insights/insights-runtime-extractor-mv2qq a914112185ee40cee3e9a43fd221947f0c0cd0d3574d65da9e6537e72500c6f0 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-mv2qq a914112185ee40cee3e9a43fd221947f0c0cd0d3574d65da9e6537e72500c6f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:09:3e:5c:7f:a9\",\"name\":\"a914112185ee40c\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d4e0a8c3-0f07-4ebe-922b-d2c9ea152824\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:30.498833 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:30.498852 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:30.500285 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:30.500306 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:30.500314 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0326 23:36:30.558634 4021 obj_retry.go:464] Detected object openshift-image-registry/image-registry-66c8fd6f5d-kfmps of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:36:30.559239 4021 obj_retry.go:464] Detected object openshift-image-registry/image-registry-66c8fd6f5d-kfmps of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:36:30.572159 4021 obj_retry.go:458] Detected object openshift-image-registry/image-registry-66c8fd6f5d-kfmps of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:36:30.572179 4021 obj_retry.go:458] Detected object openshift-image-registry/image-registry-66c8fd6f5d-kfmps of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:36:30.583024 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-66c8fd6f5d-kfmps I0326 23:36:30.583047 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-66c8fd6f5d-kfmps I0326 23:36:30.704269 4021 cni.go:403] [openshift-ingress/router-default-56b54dbfd-7rf85 efc6f970e7427435615ecc0736b7fdc4069da397544acea41129939f9404ec51 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-56b54dbfd-7rf85 efc6f970e7427435615ecc0736b7fdc4069da397544acea41129939f9404ec51 network default NAD default] I0326 23:36:30.708822 4021 cni.go:403] [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default] I0326 23:36:30.713326 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-56b54dbfd-7rf85, hostIfaceName: efc6f970e742743, network: default, NAD default, SandboxID: "efc6f970e7427435615ecc0736b7fdc4069da397544acea41129939f9404ec51", PCI device ID: , UID: "03c3ab17-746d-48eb-a798-df62c9337b67", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0326 23:36:30.718068 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-764b98bc4f-cmq8t, hostIfaceName: 2dc05530528ee1c, network: default, NAD default, SandboxID: "2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0", PCI device ID: , UID: "33651f2b-1557-45ea-b815-7859964a326a", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0326 23:36:30.753709 4021 cni.go:424] [openshift-ingress/router-default-56b54dbfd-7rf85 efc6f970e7427435615ecc0736b7fdc4069da397544acea41129939f9404ec51 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-56b54dbfd-7rf85 efc6f970e7427435615ecc0736b7fdc4069da397544acea41129939f9404ec51 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:92:1a:82:58:6b\",\"name\":\"efc6f970e742743\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ac75e619-d081-4838-bf50-df5980aea0e7\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:36:30.772148 4021 cni.go:424] [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:3c:6d:95:a6:a9\",\"name\":\"2dc05530528ee1c\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a17daee8-cb1f-4582-b2cb-aa0483fd0964\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:31.076438 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.076464 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.076470 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.077656 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.077672 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.104991 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.105007 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.107213 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.107226 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.107230 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0326 23:36:31.130587 4021 obj_retry.go:464] Detected object openshift-image-registry/image-registry-6b5d664bf7-98zz6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:36:31.130632 4021 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6b5d664bf7-98zz6 I0326 23:36:31.131860 4021 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6b5d664bf7-98zz6, ips: 10.129.0.6 I0326 23:36:31.131883 4021 obj_retry.go:464] Detected object openshift-image-registry/image-registry-6b5d664bf7-98zz6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:36:31.139064 4021 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-69dd55cf89-m99gz] creating logical port openshift-image-registry_image-registry-69dd55cf89-m99gz for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:31.139212 4021 kube.go:256] Updating pod openshift-image-registry/image-registry-69dd55cf89-m99gz I0326 23:36:31.145149 4021 obj_retry.go:458] Detected object openshift-image-registry/image-registry-6b5d664bf7-98zz6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:36:31.145169 4021 obj_retry.go:458] Detected object openshift-image-registry/image-registry-6b5d664bf7-98zz6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:36:31.151475 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6b5d664bf7-98zz6 I0326 23:36:31.151497 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6b5d664bf7-98zz6 I0326 23:36:31.159899 4021 pod.go:62] [openshift-image-registry/image-registry-69dd55cf89-m99gz] pod update took 20.725441ms I0326 23:36:31.159917 4021 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-69dd55cf89-m99gz] addLogicalPort annotation time took 20.749408ms I0326 23:36:31.161115 4021 pods.go:271] [openshift-image-registry/image-registry-69dd55cf89-m99gz] addLogicalPort took 22.066297ms, libovsdb time 899.563µs I0326 23:36:31.496530 4021 cni.go:403] [openshift-image-registry/image-registry-69dd55cf89-m99gz cd127d5c4b7d58e95019162f38bb8cc6d952546150fc3d2cfb64beb2f22fd572 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-69dd55cf89-m99gz cd127d5c4b7d58e95019162f38bb8cc6d952546150fc3d2cfb64beb2f22fd572 network default NAD default] I0326 23:36:31.505440 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-69dd55cf89-m99gz, hostIfaceName: cd127d5c4b7d58e, network: default, NAD default, SandboxID: "cd127d5c4b7d58e95019162f38bb8cc6d952546150fc3d2cfb64beb2f22fd572", PCI device ID: , UID: "dc254272-492d-4763-87c9-2ad5d268c190", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] W0326 23:36:31.539663 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.539686 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.539688 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.539707 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.539726 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.539734 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:31.542286 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.542305 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.542310 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.542329 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.542336 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:31.542360 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0326 23:36:31.549988 4021 cni.go:424] [openshift-image-registry/image-registry-69dd55cf89-m99gz cd127d5c4b7d58e95019162f38bb8cc6d952546150fc3d2cfb64beb2f22fd572 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-69dd55cf89-m99gz cd127d5c4b7d58e95019162f38bb8cc6d952546150fc3d2cfb64beb2f22fd572 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:5e:08:d4:26:41\",\"name\":\"cd127d5c4b7d58e\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8b3b4b00-8907-4b3c-b3cb-e97843cc6b26\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:31.578652 4021 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:32.086596 4021 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:32.117822 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:36:32.117843 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0326 23:36:32.190204 4021 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:36:32.202707 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:32.202723 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:36:32.202729 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0326 23:36:32.298161 4021 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:36:32.484686 4021 base_network_controller_pods.go:487] [default/openshift-console/console-574dd8946c-62j7h] creating logical port openshift-console_console-574dd8946c-62j7h for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:32.484873 4021 kube.go:256] Updating pod openshift-console/console-574dd8946c-62j7h I0326 23:36:32.502163 4021 pod.go:62] [openshift-console/console-574dd8946c-62j7h] pod update took 17.323352ms I0326 23:36:32.502185 4021 base_network_controller_pods.go:937] [default/openshift-console/console-574dd8946c-62j7h] addLogicalPort annotation time took 17.352745ms I0326 23:36:32.503358 4021 pods.go:271] [openshift-console/console-574dd8946c-62j7h] addLogicalPort took 18.677901ms, libovsdb time 841.398µs I0326 23:36:32.842209 4021 cni.go:403] [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default] ADD starting CNI request [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default] I0326 23:36:32.850437 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-574dd8946c-62j7h, hostIfaceName: 3075d2a9a73c7f8, network: default, NAD default, SandboxID: "3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8", PCI device ID: , UID: "7424f5c6-78d3-47bb-8f56-0eafefbb135d", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0326 23:36:32.891693 4021 cni.go:424] [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default] ADD finished CNI request [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:c3:de:d4:f4:b4\",\"name\":\"3075d2a9a73c7f8\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/94005ff4-2fdc-49ae-9711-512e9f7cf84e\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:35.532188 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:36:36.102159 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:36:36.594619 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:36:37.243932 4021 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-c56ff685-bs2fd] creating logical port openshift-monitoring_prometheus-operator-c56ff685-bs2fd for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:37.244032 4021 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-c56ff685-bs2fd I0326 23:36:37.258363 4021 pod.go:62] [openshift-monitoring/prometheus-operator-c56ff685-bs2fd] pod update took 14.360768ms I0326 23:36:37.258383 4021 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-c56ff685-bs2fd] addLogicalPort annotation time took 14.386289ms I0326 23:36:37.259530 4021 pods.go:271] [openshift-monitoring/prometheus-operator-c56ff685-bs2fd] addLogicalPort took 15.617837ms, libovsdb time 836.502µs I0326 23:36:37.601776 4021 cni.go:403] [openshift-monitoring/prometheus-operator-c56ff685-bs2fd 6d189606f749f12fa29fabe948b2abb243ef87a911a96822a9efffb0d4e35139 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-c56ff685-bs2fd 6d189606f749f12fa29fabe948b2abb243ef87a911a96822a9efffb0d4e35139 network default NAD default] I0326 23:36:37.610366 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-c56ff685-bs2fd, hostIfaceName: 6d189606f749f12, network: default, NAD default, SandboxID: "6d189606f749f12fa29fabe948b2abb243ef87a911a96822a9efffb0d4e35139", PCI device ID: , UID: "6e3e0cf8-1dcc-4742-a8f8-5d0f2fd32f72", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0326 23:36:37.651968 4021 cni.go:424] [openshift-monitoring/prometheus-operator-c56ff685-bs2fd 6d189606f749f12fa29fabe948b2abb243ef87a911a96822a9efffb0d4e35139 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-c56ff685-bs2fd 6d189606f749f12fa29fabe948b2abb243ef87a911a96822a9efffb0d4e35139 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:2f:8c:3c:98:03\",\"name\":\"6d189606f749f12\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9e4cafd-dba8-4597-959e-49097a7dfa3a\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:36:40.697666 4021 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz] creating logical port openshift-monitoring_openshift-state-metrics-bc865d6bb-t97cz for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:40.697773 4021 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz I0326 23:36:40.729426 4021 pod.go:62] [openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz] pod update took 31.676731ms I0326 23:36:40.729460 4021 base_network_controller_pods.go:937] [default/openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz] addLogicalPort annotation time took 31.716723ms I0326 23:36:40.730863 4021 pods.go:271] [openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz] addLogicalPort took 33.215965ms, libovsdb time 985.115µs I0326 23:36:41.053672 4021 cni.go:403] [openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz 9e8978468b24924d06d8f3c84b4fd5cfc0be495e1103832cd15f9816f2f9147e network default NAD default] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz 9e8978468b24924d06d8f3c84b4fd5cfc0be495e1103832cd15f9816f2f9147e network default NAD default] I0326 23:36:41.062856 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-bc865d6bb-t97cz, hostIfaceName: 9e8978468b24924, network: default, NAD default, SandboxID: "9e8978468b24924d06d8f3c84b4fd5cfc0be495e1103832cd15f9816f2f9147e", PCI device ID: , UID: "5bd840e2-6999-444b-80b7-ea5e004ba158", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0326 23:36:41.113018 4021 cni.go:424] [openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz 9e8978468b24924d06d8f3c84b4fd5cfc0be495e1103832cd15f9816f2f9147e network default NAD default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-bc865d6bb-t97cz 9e8978468b24924d06d8f3c84b4fd5cfc0be495e1103832cd15f9816f2f9147e network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:87:a0:e6:ee:0d\",\"name\":\"9e8978468b24924\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d94779a2-d492-4fb3-8c9f-6a5c1e589949\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:36:41.637527 4021 cni.go:403] [openshift-dns/dns-default-tkng8 d8d67033d409d1f401992c3c50ba626d7c7ebdcae9ef8b0e67f7d24b93155775 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-tkng8 d8d67033d409d1f401992c3c50ba626d7c7ebdcae9ef8b0e67f7d24b93155775 network default NAD default] I0326 23:36:41.646168 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-tkng8, hostIfaceName: d8d67033d409d1f, network: default, NAD default, SandboxID: "d8d67033d409d1f401992c3c50ba626d7c7ebdcae9ef8b0e67f7d24b93155775", PCI device ID: , UID: "97f405fd-7353-4ed5-889a-bd6b7f66b9d9", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0326 23:36:41.692054 4021 cni.go:424] [openshift-dns/dns-default-tkng8 d8d67033d409d1f401992c3c50ba626d7c7ebdcae9ef8b0e67f7d24b93155775 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-tkng8 d8d67033d409d1f401992c3c50ba626d7c7ebdcae9ef8b0e67f7d24b93155775 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:77:7c:74:e6:27\",\"name\":\"d8d67033d409d1f\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/faf3a1f9-9439-4287-8981-ef1cc874346d\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:36:41.832741 4021 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:41.832856 4021 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0326 23:36:41.854294 4021 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 21.478073ms I0326 23:36:41.854315 4021 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 21.504839ms I0326 23:36:41.855617 4021 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 22.895246ms, libovsdb time 919.178µs W0326 23:36:41.878222 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:36:42.194977 4021 cni.go:403] [openshift-monitoring/alertmanager-main-1 3e3e57e9fa49ecd3b16ba1b88141b912a277f5a60eda8e3a36183c8c6bc0d306 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 3e3e57e9fa49ecd3b16ba1b88141b912a277f5a60eda8e3a36183c8c6bc0d306 network default NAD default] I0326 23:36:42.203980 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 3e3e57e9fa49ecd, network: default, NAD default, SandboxID: "3e3e57e9fa49ecd3b16ba1b88141b912a277f5a60eda8e3a36183c8c6bc0d306", PCI device ID: , UID: "49bb18e7-356a-436e-885b-f15cb257cb93", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0326 23:36:42.250019 4021 cni.go:424] [openshift-monitoring/alertmanager-main-1 3e3e57e9fa49ecd3b16ba1b88141b912a277f5a60eda8e3a36183c8c6bc0d306 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 3e3e57e9fa49ecd3b16ba1b88141b912a277f5a60eda8e3a36183c8c6bc0d306 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:3c:50:bc:f4:39\",\"name\":\"3e3e57e9fa49ecd\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2b3db40-d97d-4508-a021-ba0fa167097b\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:36:42.812308 4021 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-5885d449c7-zhpdv] creating logical port openshift-monitoring_thanos-querier-5885d449c7-zhpdv for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:42.812435 4021 kube.go:256] Updating pod openshift-monitoring/thanos-querier-5885d449c7-zhpdv I0326 23:36:42.832864 4021 pod.go:62] [openshift-monitoring/thanos-querier-5885d449c7-zhpdv] pod update took 20.470604ms I0326 23:36:42.832888 4021 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-5885d449c7-zhpdv] addLogicalPort annotation time took 20.499091ms I0326 23:36:42.842181 4021 pods.go:271] [openshift-monitoring/thanos-querier-5885d449c7-zhpdv] addLogicalPort took 29.888796ms, libovsdb time 6.197076ms W0326 23:36:43.227703 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:36:43.476551 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:43.476574 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:36:43.628291 4021 cni.go:403] [openshift-monitoring/thanos-querier-5885d449c7-zhpdv 2a0a612b48a996142f079a259563578d35242d6824ed028397596a3091b385f6 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-5885d449c7-zhpdv 2a0a612b48a996142f079a259563578d35242d6824ed028397596a3091b385f6 network default NAD default] I0326 23:36:43.639380 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5885d449c7-zhpdv, hostIfaceName: 2a0a612b48a9961, network: default, NAD default, SandboxID: "2a0a612b48a996142f079a259563578d35242d6824ed028397596a3091b385f6", PCI device ID: , UID: "d9eea6ca-45b9-4c32-a926-a47c62fc49e0", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0326 23:36:43.691523 4021 cni.go:424] [openshift-monitoring/thanos-querier-5885d449c7-zhpdv 2a0a612b48a996142f079a259563578d35242d6824ed028397596a3091b385f6 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-5885d449c7-zhpdv 2a0a612b48a996142f079a259563578d35242d6824ed028397596a3091b385f6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:64:7b:a8:d9:3b\",\"name\":\"2a0a612b48a9961\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f46a863-77e1-413d-bc9e-deadf6855b4b\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:44.189074 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:36:44.189100 4021 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:36:44.189106 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:36:45.621377 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:36:46.717236 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:36:46.717259 4021 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:36:46.717266 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:36:47.153263 4021 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:36:47.153435 4021 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0326 23:36:47.174314 4021 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 20.920532ms I0326 23:36:47.174435 4021 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 21.045576ms I0326 23:36:47.175953 4021 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 22.709402ms, libovsdb time 1.0549ms W0326 23:36:47.226757 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:47.226781 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:47.226789 4021 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:47.226795 4021 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:36:47.515729 4021 cni.go:403] [openshift-monitoring/prometheus-k8s-0 73ccf88fa501ecc57dc8e8ac89ffb03726f1abbad48c4b3d29cece289b08dcdc network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 73ccf88fa501ecc57dc8e8ac89ffb03726f1abbad48c4b3d29cece289b08dcdc network default NAD default] I0326 23:36:47.524971 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 73ccf88fa501ecc, network: default, NAD default, SandboxID: "73ccf88fa501ecc57dc8e8ac89ffb03726f1abbad48c4b3d29cece289b08dcdc", PCI device ID: , UID: "178b265b-a6e6-4f14-95a2-2b9c0b7296c4", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0326 23:36:47.575660 4021 cni.go:424] [openshift-monitoring/prometheus-k8s-0 73ccf88fa501ecc57dc8e8ac89ffb03726f1abbad48c4b3d29cece289b08dcdc network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 73ccf88fa501ecc57dc8e8ac89ffb03726f1abbad48c4b3d29cece289b08dcdc network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:db:87:72:6c:b0\",\"name\":\"73ccf88fa501ecc\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19d469af-a2d1-4169-9c66-cb839a4460ad\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:36:48.202232 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:36:48.202361 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:36:48.675750 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:36:48.853830 4021 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-64489c4789-qpz86 IP address from the namespace address-set, err: pod openshift-console/console-64489c4789-qpz86: no pod IPs found W0326 23:36:49.602861 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:36:49.668336 4021 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:49.668374 4021 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:49.668381 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:49.668387 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:49.695507 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:36:49.695523 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:36:52.649277 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:52.649304 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:52.694596 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:52.694622 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:52.829736 4021 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-6785d58857-p986s IP address from the namespace address-set, err: pod openshift-console/console-6785d58857-p986s: no pod IPs found W0326 23:36:52.829793 4021 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-6785d58857-zhh6j IP address from the namespace address-set, err: pod openshift-console/console-6785d58857-zhh6j: no pod IPs found W0326 23:36:53.200368 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:53.200391 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:53.200400 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:53.200406 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:36:54.103717 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:36:54.157252 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:36:54.157269 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:36:54.157275 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default I0326 23:37:01.187272 4021 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:37:06.765650 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0326 23:37:07.209977 4021 obj_retry.go:464] Detected object openshift-console/console-574dd8946c-trl44 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:07.210552 4021 obj_retry.go:464] Detected object openshift-console/console-574dd8946c-trl44 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:07.795029 4021 obj_retry.go:458] Detected object openshift-console/console-574dd8946c-trl44 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:07.795051 4021 obj_retry.go:458] Detected object openshift-console/console-574dd8946c-trl44 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:07.801097 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-574dd8946c-trl44 I0326 23:37:07.801117 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-574dd8946c-trl44 W0326 23:37:11.805582 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:37:12.212250 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.212275 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.212283 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.212300 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.212307 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.212314 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.256940 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.256959 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:37:12.256966 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:37:15.926492 4021 obj_retry.go:464] Detected object openshift-console/console-64489c4789-gglzh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:15.927074 4021 obj_retry.go:464] Detected object openshift-console/console-64489c4789-gglzh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:16.738986 4021 obj_retry.go:458] Detected object openshift-console/console-64489c4789-gglzh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:16.739007 4021 obj_retry.go:458] Detected object openshift-console/console-64489c4789-gglzh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:16.741752 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-64489c4789-gglzh I0326 23:37:16.741777 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-64489c4789-gglzh I0326 23:37:18.020294 4021 obj_retry.go:464] Detected object openshift-image-registry/image-registry-6b5d664bf7-xfz58 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:18.021164 4021 obj_retry.go:464] Detected object openshift-image-registry/image-registry-6b5d664bf7-xfz58 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:18.725512 4021 obj_retry.go:458] Detected object openshift-image-registry/image-registry-6b5d664bf7-xfz58 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:18.725534 4021 obj_retry.go:458] Detected object openshift-image-registry/image-registry-6b5d664bf7-xfz58 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:18.728550 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6b5d664bf7-xfz58 I0326 23:37:18.728566 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6b5d664bf7-xfz58 W0326 23:37:21.328112 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0326 23:37:21.328118 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0326 23:37:21.328290 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0326 23:37:22.331606 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0326 23:37:43.028198 4021 cni.go:403] [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default] DEL starting CNI request [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default] I0326 23:37:43.104890 4021 cni.go:424] [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default] DEL finished CNI request [openshift-console/console-574dd8946c-62j7h 3075d2a9a73c7f8d20d6769aa3ba0c026ee9d2dedc1872164e258cdad80f59b8 network default NAD default], result "{}", err I0326 23:37:43.169690 4021 obj_retry.go:464] Detected object openshift-console/console-574dd8946c-62j7h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:43.169726 4021 pods.go:176] Deleting pod: openshift-console/console-574dd8946c-62j7h I0326 23:37:43.171150 4021 pods.go:236] Attempting to release IPs for pod: openshift-console/console-574dd8946c-62j7h, ips: 10.129.0.16 I0326 23:37:43.171180 4021 obj_retry.go:464] Detected object openshift-console/console-574dd8946c-62j7h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:43.407492 4021 obj_retry.go:458] Detected object openshift-console/console-574dd8946c-62j7h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:43.407523 4021 obj_retry.go:458] Detected object openshift-console/console-574dd8946c-62j7h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:43.412895 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-574dd8946c-62j7h I0326 23:37:43.412917 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-574dd8946c-62j7h I0326 23:37:47.195234 4021 cni.go:403] [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default] DEL starting CNI request [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default] I0326 23:37:47.267901 4021 cni.go:424] [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default] DEL finished CNI request [openshift-ingress/router-default-764b98bc4f-cmq8t 2dc05530528ee1c2dcc1adaa36e14f559704f90d58806f5c36fc0f5386217fb0 network default NAD default], result "{}", err I0326 23:37:47.314246 4021 obj_retry.go:464] Detected object openshift-ingress/router-default-764b98bc4f-cmq8t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:47.314282 4021 pods.go:176] Deleting pod: openshift-ingress/router-default-764b98bc4f-cmq8t I0326 23:37:47.315789 4021 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-764b98bc4f-cmq8t, ips: 10.129.0.13 I0326 23:37:47.315823 4021 obj_retry.go:464] Detected object openshift-ingress/router-default-764b98bc4f-cmq8t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:37:47.431846 4021 obj_retry.go:458] Detected object openshift-ingress/router-default-764b98bc4f-cmq8t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:47.431875 4021 obj_retry.go:458] Detected object openshift-ingress/router-default-764b98bc4f-cmq8t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:47.437482 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-764b98bc4f-cmq8t I0326 23:37:47.437497 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-764b98bc4f-cmq8t W0326 23:37:47.982406 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:37:47.982429 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:37:47.982440 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:37:47.982445 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:37:48.478948 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:37:48.478973 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:38:01.638791 4021 cni.go:403] [openshift-multus/network-metrics-daemon-twhht 76af6c338090f0e5b9377c8da980f3fea8f3743a81af137c7102da68d8323c94 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-twhht 76af6c338090f0e5b9377c8da980f3fea8f3743a81af137c7102da68d8323c94 network default NAD default] I0326 23:38:01.647692 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-twhht, hostIfaceName: 76af6c338090f0e, network: default, NAD default, SandboxID: "76af6c338090f0e5b9377c8da980f3fea8f3743a81af137c7102da68d8323c94", PCI device ID: , UID: "94d6d5c0-8786-4b0a-9dba-0a02fb6cad85", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0326 23:38:01.690432 4021 cni.go:424] [openshift-multus/network-metrics-daemon-twhht 76af6c338090f0e5b9377c8da980f3fea8f3743a81af137c7102da68d8323c94 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-twhht 76af6c338090f0e5b9377c8da980f3fea8f3743a81af137c7102da68d8323c94 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:da:06:f8:7f:b6\",\"name\":\"76af6c338090f0e\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ccae7f3e-bab8-45f0-bb18-e65c543377d9\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:38:16.067439 4021 base_network_controller_pods.go:487] [default/openshift-console/console-797f4c6847-2b5p5] creating logical port openshift-console_console-797f4c6847-2b5p5 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:38:16.067560 4021 kube.go:256] Updating pod openshift-console/console-797f4c6847-2b5p5 I0326 23:38:16.093335 4021 pod.go:62] [openshift-console/console-797f4c6847-2b5p5] pod update took 25.807132ms I0326 23:38:16.093384 4021 base_network_controller_pods.go:937] [default/openshift-console/console-797f4c6847-2b5p5] addLogicalPort annotation time took 25.86138ms I0326 23:38:16.094756 4021 pods.go:271] [openshift-console/console-797f4c6847-2b5p5] addLogicalPort took 27.336898ms, libovsdb time 1.050499ms I0326 23:38:16.430161 4021 cni.go:403] [openshift-console/console-797f4c6847-2b5p5 cb2ce533befa3dfac1edfd7120ccad980fe7b815356ddbd17e8c6160b9c97bf2 network default NAD default] ADD starting CNI request [openshift-console/console-797f4c6847-2b5p5 cb2ce533befa3dfac1edfd7120ccad980fe7b815356ddbd17e8c6160b9c97bf2 network default NAD default] I0326 23:38:16.438614 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-797f4c6847-2b5p5, hostIfaceName: cb2ce533befa3df, network: default, NAD default, SandboxID: "cb2ce533befa3dfac1edfd7120ccad980fe7b815356ddbd17e8c6160b9c97bf2", PCI device ID: , UID: "dfcb05ac-5c8f-4644-b9bb-da49b9263d1a", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0326 23:38:16.481075 4021 cni.go:424] [openshift-console/console-797f4c6847-2b5p5 cb2ce533befa3dfac1edfd7120ccad980fe7b815356ddbd17e8c6160b9c97bf2 network default NAD default] ADD finished CNI request [openshift-console/console-797f4c6847-2b5p5 cb2ce533befa3dfac1edfd7120ccad980fe7b815356ddbd17e8c6160b9c97bf2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:3a:a1:b4:52:03\",\"name\":\"cb2ce533befa3df\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a132d09-803b-4369-8d1e-0a5cf84be5eb\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:38:19.312024 4021 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0326 23:38:26.608290 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:38:41.378821 4021 obj_retry.go:464] Detected object openshift-console/console-9dc8dc589-plgzs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:38:41.379432 4021 obj_retry.go:464] Detected object openshift-console/console-9dc8dc589-plgzs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:38:42.176895 4021 obj_retry.go:458] Detected object openshift-console/console-9dc8dc589-plgzs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:38:42.176916 4021 obj_retry.go:458] Detected object openshift-console/console-9dc8dc589-plgzs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:38:42.179676 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-9dc8dc589-plgzs I0326 23:38:42.179688 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-9dc8dc589-plgzs I0326 23:38:51.926555 4021 obj_retry.go:464] Detected object openshift-console/console-9dc8dc589-s78mx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:38:51.927134 4021 obj_retry.go:464] Detected object openshift-console/console-9dc8dc589-s78mx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:38:52.086656 4021 obj_retry.go:458] Detected object openshift-console/console-9dc8dc589-s78mx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:38:52.086688 4021 obj_retry.go:458] Detected object openshift-console/console-9dc8dc589-s78mx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:38:52.091080 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-9dc8dc589-s78mx I0326 23:38:52.091104 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-9dc8dc589-s78mx I0326 23:39:33.671807 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 W0326 23:39:52.325862 4021 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:40:00.470360 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:00.470914 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:01.369656 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:01.369709 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:01.392748 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:01.392769 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:01.395927 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn I0326 23:40:01.395950 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-fg6kn I0326 23:40:03.367357 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:40:03.367420 4021 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-57thh IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-57thh: no pod IPs found I0326 23:40:03.611672 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:03.612212 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:04.505723 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:04.505748 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:07.662529 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0326 23:40:09.577413 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=117 I0326 23:40:15.578526 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0326 23:40:18.683289 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:40:18.688530 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0326 23:40:35.577946 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0326 23:40:43.695075 4021 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:40:48.679785 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0326 23:40:52.919613 4021 namespace.go:98] [tsf] adding namespace I0326 23:40:52.920600 4021 namespace.go:102] [tsf] adding namespace took 958.731µs I0326 23:40:52.934973 4021 namespace.go:142] [tsf] updating namespace I0326 23:40:52.944530 4021 namespace.go:142] [tsf] updating namespace I0326 23:40:56.602632 4021 namespace.go:98] [rhbk-operator] adding namespace I0326 23:40:56.603407 4021 namespace.go:102] [rhbk-operator] adding namespace took 759.181µs I0326 23:40:56.609510 4021 namespace.go:98] [tsf-keycloak] adding namespace I0326 23:40:56.609588 4021 namespace.go:98] [rhtpa-operator] adding namespace I0326 23:40:56.610218 4021 namespace.go:102] [tsf-keycloak] adding namespace took 693.093µs I0326 23:40:56.610977 4021 namespace.go:102] [rhtpa-operator] adding namespace took 1.3739ms I0326 23:40:56.617404 4021 namespace.go:98] [konflux-operator] adding namespace I0326 23:40:56.617526 4021 namespace.go:98] [cert-manager-operator] adding namespace I0326 23:40:56.617609 4021 namespace.go:98] [tsf-tas] adding namespace I0326 23:40:56.617660 4021 namespace.go:98] [konflux-ui] adding namespace I0326 23:40:56.617812 4021 namespace.go:98] [tsf-tpa] adding namespace I0326 23:40:56.618106 4021 namespace.go:102] [konflux-operator] adding namespace took 685.183µs I0326 23:40:56.618647 4021 namespace.go:102] [cert-manager-operator] adding namespace took 1.107277ms I0326 23:40:56.619137 4021 namespace.go:102] [tsf-tas] adding namespace took 1.51356ms I0326 23:40:56.619615 4021 namespace.go:102] [konflux-ui] adding namespace took 1.944022ms I0326 23:40:56.620116 4021 namespace.go:102] [tsf-tpa] adding namespace took 2.285889ms I0326 23:40:56.620456 4021 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:56.634258 4021 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:56.645538 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:56.655812 4021 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:56.656723 4021 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:56.656770 4021 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:56.656838 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:56.668950 4021 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:56.672321 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:56.675139 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:56.677311 4021 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:56.677324 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:56.704251 4021 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:56.704394 4021 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:56.710169 4021 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:56.718810 4021 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:56.730782 4021 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:56.740233 4021 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:56.751284 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:56.773531 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:56.773552 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:56.783069 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:56.786910 4021 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:56.801331 4021 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:57.990635 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:57.990704 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:04.150066 4021 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:04.150642 4021 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:04.150671 4021 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:04.150679 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:04.150692 4021 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:04.150696 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:05.175277 4021 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:05.175300 4021 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:05.175315 4021 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:05.175332 4021 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfbshfb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:11.852126 4021 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.852705 4021 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.852734 4021 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.852744 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:11.852759 4021 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.852763 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:11.907984 4021 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.908471 4021 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.908493 4021 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.908501 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:11.908514 4021 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.908519 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:11.940101 4021 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.940531 4021 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.940551 4021 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.940558 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:11.940571 4021 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.940574 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:11.966437 4021 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.966816 4021 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.966836 4021 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.966843 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:11.966854 4021 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:11.966858 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:12.882561 4021 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.882589 4021 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.882603 4021 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.882624 4021 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7p4nkc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.935714 4021 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.935736 4021 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.935747 4021 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.935760 4021 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafpt4qv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.963838 4021 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.963861 4021 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.963872 4021 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.963885 4021 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1xmbw5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.991640 4021 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.991658 4021 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.991667 4021 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:12.991679 4021 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8jpd6l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:19.030804 4021 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-bs4xs] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-bs4xs for pod on switch ip-10-0-2-65.ec2.internal I0326 23:41:19.030911 4021 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-bs4xs I0326 23:41:19.049142 4021 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-bs4xs] pod update took 18.249527ms I0326 23:41:19.049173 4021 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-bs4xs] addLogicalPort annotation time took 18.286955ms I0326 23:41:19.050374 4021 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-bs4xs] addLogicalPort took 19.592432ms, libovsdb time 844.64µs I0326 23:41:19.409659 4021 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-bs4xs a17100b2c1d2d946ff6dd0211b2caeaa6f4c78d9af912a01ffa6954c23e567fa network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-bs4xs a17100b2c1d2d946ff6dd0211b2caeaa6f4c78d9af912a01ffa6954c23e567fa network default NAD default] I0326 23:41:19.417752 4021 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-bs4xs, hostIfaceName: a17100b2c1d2d94, network: default, NAD default, SandboxID: "a17100b2c1d2d946ff6dd0211b2caeaa6f4c78d9af912a01ffa6954c23e567fa", PCI device ID: , UID: "afb8a8ba-b57b-4a2a-ae2b-989a5cfc4faa", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0326 23:41:19.425555 4021 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-wghj6 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:41:19.425686 4021 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 I0326 23:41:19.448827 4021 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6] pod update took 23.190689ms I0326 23:41:19.448852 4021 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6] addLogicalPort annotation time took 23.221266ms I0326 23:41:19.450234 4021 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6] addLogicalPort took 24.69837ms, libovsdb time 997.025µs I0326 23:41:19.463999 4021 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-bs4xs a17100b2c1d2d946ff6dd0211b2caeaa6f4c78d9af912a01ffa6954c23e567fa network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-bs4xs a17100b2c1d2d946ff6dd0211b2caeaa6f4c78d9af912a01ffa6954c23e567fa network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:63:57:65:bf:6a\",\"name\":\"a17100b2c1d2d94\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f2a478d-710b-43c1-b56f-979c3c828e21\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:41:19.786568 4021 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default] I0326 23:41:19.794761 4021 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-wghj6, hostIfaceName: 2ff0f62f7f7029b, network: default, NAD default, SandboxID: "2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917", PCI device ID: , UID: "b88ee88d-9ac6-40db-b2f8-585823ea8beb", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0326 23:41:19.837169 4021 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:35:ad:54:24:87\",\"name\":\"2ff0f62f7f7029b\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ec1dd778-400d-4139-8cf7-65960ac971f6\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:41:20.892993 4021 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:20.893655 4021 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:20.893690 4021 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:20.893701 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:20.893719 4021 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:20.893725 4021 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:21.937632 4021 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:21.937661 4021 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:21.937680 4021 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:21.937698 4021 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5wndnm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:41:22.288420 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:22.479381 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:22.488447 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:22.498828 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:22.505491 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:41:29.472676 4021 namespace.go:142] [openshift-operators] updating namespace I0326 23:41:29.492619 4021 namespace.go:142] [openshift-operators] updating namespace W0326 23:41:31.863561 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0326 23:41:33.298898 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:41:36.461676 4021 namespace.go:142] [openshift-operators] updating namespace W0326 23:41:46.147326 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:41:46.313296 4021 namespace.go:98] [trusted-artifact-signer] adding namespace I0326 23:41:46.314185 4021 namespace.go:102] [trusted-artifact-signer] adding namespace took 862.878µs I0326 23:41:46.358870 4021 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:46.385309 4021 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:46.512358 4021 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-6fbm7 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:41:46.512444 4021 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7 I0326 23:41:46.532138 4021 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7] pod update took 19.712881ms I0326 23:41:46.532165 4021 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7] addLogicalPort annotation time took 19.742535ms I0326 23:41:46.533454 4021 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7] addLogicalPort took 21.125468ms, libovsdb time 941.858µs I0326 23:41:46.967658 4021 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7 4bc9a8cbbb4efed585cf137d521d516e021b94c032a7f1e07894a2bc8be49ed6 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7 4bc9a8cbbb4efed585cf137d521d516e021b94c032a7f1e07894a2bc8be49ed6 network default NAD default] I0326 23:41:46.976617 4021 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-6fbm7, hostIfaceName: 4bc9a8cbbb4efed, network: default, NAD default, SandboxID: "4bc9a8cbbb4efed585cf137d521d516e021b94c032a7f1e07894a2bc8be49ed6", PCI device ID: , UID: "eb8f647b-23ca-4809-b9e2-a3fad4b0a70d", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0326 23:41:47.020570 4021 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7 4bc9a8cbbb4efed585cf137d521d516e021b94c032a7f1e07894a2bc8be49ed6 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-6fbm7 4bc9a8cbbb4efed585cf137d521d516e021b94c032a7f1e07894a2bc8be49ed6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:f8:e7:c5:e8:23\",\"name\":\"4bc9a8cbbb4efed\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/066c429c-73f1-4d97-be66-3b3d1198411b\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:41:48.700484 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0326 23:41:51.258065 4021 namespace.go:98] [openshift-pipelines] adding namespace I0326 23:41:51.259131 4021 namespace.go:102] [openshift-pipelines] adding namespace took 1.044778ms I0326 23:41:51.276326 4021 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:41:52.254362 4021 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:52.648019 4021 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:52.701512 4021 namespace.go:98] [cert-manager] adding namespace I0326 23:41:52.702324 4021 namespace.go:102] [cert-manager] adding namespace took 786.688µs I0326 23:41:52.731441 4021 namespace.go:142] [cert-manager] updating namespace I0326 23:41:52.749035 4021 namespace.go:142] [cert-manager] updating namespace I0326 23:41:52.874474 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:53.093944 4021 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-5j6nc for pod on switch ip-10-0-2-65.ec2.internal I0326 23:41:53.094082 4021 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc I0326 23:41:53.114225 4021 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc] pod update took 20.16919ms I0326 23:41:53.114252 4021 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc] addLogicalPort annotation time took 20.199845ms I0326 23:41:53.115651 4021 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc] addLogicalPort took 21.742014ms, libovsdb time 1.014691ms I0326 23:41:53.170324 4021 namespace.go:142] [open-cluster-management-2p9n1rl9ff1n78gavi22o5cr9k7iopuv] updating namespace I0326 23:41:53.456580 4021 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc 8dfbe37c838a901ffa2dc667f3fdf9dcf692a0046abf5d3c562c36067cf32ec0 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc 8dfbe37c838a901ffa2dc667f3fdf9dcf692a0046abf5d3c562c36067cf32ec0 network default NAD default] I0326 23:41:53.465669 4021 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-5j6nc, hostIfaceName: 8dfbe37c838a901, network: default, NAD default, SandboxID: "8dfbe37c838a901ffa2dc667f3fdf9dcf692a0046abf5d3c562c36067cf32ec0", PCI device ID: , UID: "d0864c86-183c-4710-9d6a-0146376a54d2", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0326 23:41:53.514189 4021 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc 8dfbe37c838a901ffa2dc667f3fdf9dcf692a0046abf5d3c562c36067cf32ec0 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-5j6nc 8dfbe37c838a901ffa2dc667f3fdf9dcf692a0046abf5d3c562c36067cf32ec0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:f1:75:49:b1:b2\",\"name\":\"8dfbe37c838a901\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c0031728-2724-463e-a71c-4ae520e9510d\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:41:53.568291 4021 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:53.779808 4021 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:53.972999 4021 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:54.174225 4021 namespace.go:142] [tsf] updating namespace I0326 23:41:54.377287 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:54.574984 4021 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:54.773516 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:54.895364 4021 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:54.905035 4021 namespace.go:142] [default] updating namespace I0326 23:41:54.914074 4021 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:54.925822 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:54.936475 4021 namespace.go:142] [open-cluster-management-2p9n1rl9ff1n78gavi22o5cr9k7iopuv] updating namespace I0326 23:41:54.944701 4021 namespace.go:142] [openshift] updating namespace I0326 23:41:54.956286 4021 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:54.966317 4021 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:54.975250 4021 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:54.985647 4021 namespace.go:142] [tsf] updating namespace I0326 23:41:54.997275 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:55.010486 4021 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:55.037546 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:55.140973 4021 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:55.229859 4021 namespace.go:142] [default] updating namespace I0326 23:41:55.332150 4021 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:55.433766 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:55.532369 4021 namespace.go:142] [open-cluster-management-2p9n1rl9ff1n78gavi22o5cr9k7iopuv] updating namespace I0326 23:41:55.632136 4021 namespace.go:142] [openshift] updating namespace I0326 23:41:55.738290 4021 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:55.832807 4021 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:55.930627 4021 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:56.032329 4021 namespace.go:142] [tsf] updating namespace I0326 23:41:56.133928 4021 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:56.232030 4021 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:56.333244 4021 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:56.671931 4021 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:41:57.234555 4021 namespace.go:142] [cert-manager] updating namespace I0326 23:41:57.355478 4021 namespace.go:142] [cert-manager] updating namespace I0326 23:41:57.484312 4021 namespace.go:142] [cert-manager] updating namespace W0326 23:41:58.594294 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:41:58.594312 4021 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:42:00.118449 4021 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:00.119017 4021 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:00.141198 4021 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:00.141216 4021 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:00.146851 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0326 23:42:00.146869 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0326 23:42:04.367490 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-6j9dz for pod on switch ip-10-0-2-65.ec2.internal I0326 23:42:04.367630 4021 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz I0326 23:42:04.385841 4021 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz] pod update took 18.244368ms I0326 23:42:04.385865 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz] addLogicalPort annotation time took 18.272195ms I0326 23:42:04.387055 4021 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz] addLogicalPort took 19.581634ms, libovsdb time 837.37µs W0326 23:42:04.479368 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:42:04.479385 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:42:04.693197 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-7h6cp for pod on switch ip-10-0-2-65.ec2.internal I0326 23:42:04.693315 4021 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp I0326 23:42:04.709568 4021 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp] pod update took 16.29331ms I0326 23:42:04.709662 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp] addLogicalPort annotation time took 16.393013ms I0326 23:42:04.711055 4021 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp] addLogicalPort took 17.878168ms, libovsdb time 1.017756ms I0326 23:42:04.727981 4021 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz 5778ec83f60ae266070184884174d3a73fb2820631bffcae85deef1fb2f949f4 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz 5778ec83f60ae266070184884174d3a73fb2820631bffcae85deef1fb2f949f4 network default NAD default] I0326 23:42:04.736386 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-6j9dz, hostIfaceName: 5778ec83f60ae26, network: default, NAD default, SandboxID: "5778ec83f60ae266070184884174d3a73fb2820631bffcae85deef1fb2f949f4", PCI device ID: , UID: "a4dbc144-94aa-469e-b548-205cf434cb32", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0326 23:42:04.781761 4021 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz 5778ec83f60ae266070184884174d3a73fb2820631bffcae85deef1fb2f949f4 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-6j9dz 5778ec83f60ae266070184884174d3a73fb2820631bffcae85deef1fb2f949f4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:cd:f8:42:c1:36\",\"name\":\"5778ec83f60ae26\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c42e6b6-0e53-447f-ba37-77667a080f81\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:42:05.057725 4021 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp 7ba5fdb2ba0a59fc852a54e50a374fea958e7ff63bef3fcc85c4972996f92df2 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp 7ba5fdb2ba0a59fc852a54e50a374fea958e7ff63bef3fcc85c4972996f92df2 network default NAD default] I0326 23:42:05.066179 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-7h6cp, hostIfaceName: 7ba5fdb2ba0a59f, network: default, NAD default, SandboxID: "7ba5fdb2ba0a59fc852a54e50a374fea958e7ff63bef3fcc85c4972996f92df2", PCI device ID: , UID: "cef1d772-27ca-4284-8c5a-9480b2f78c65", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0326 23:42:05.110480 4021 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp 7ba5fdb2ba0a59fc852a54e50a374fea958e7ff63bef3fcc85c4972996f92df2 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-7h6cp 7ba5fdb2ba0a59fc852a54e50a374fea958e7ff63bef3fcc85c4972996f92df2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:46:91:06:46:39\",\"name\":\"7ba5fdb2ba0a59f\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/16c1f648-be6e-4826-8e5e-60ba67711e00\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:42:08.046163 4021 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:42:08.046195 4021 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:42:08.046204 4021 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:42:08.598220 4021 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:42:08.598244 4021 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:42:08.598250 4021 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:42:08.603001 4021 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:08.603020 4021 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:08.603025 4021 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:08.603031 4021 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:12.044037 4021 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0326 23:42:12.157251 4021 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0326 23:42:13.662152 4021 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-s55tr] creating logical port cert-manager_cert-manager-59ccb4bb6b-s55tr for pod on switch ip-10-0-2-65.ec2.internal I0326 23:42:13.662263 4021 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-s55tr I0326 23:42:13.676465 4021 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-s55tr] pod update took 14.233198ms I0326 23:42:13.676496 4021 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-s55tr] addLogicalPort annotation time took 14.2679ms I0326 23:42:13.677669 4021 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-s55tr] addLogicalPort took 15.53365ms, libovsdb time 844.008µs I0326 23:42:14.021716 4021 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-s55tr 019792f16922113b95929873c3001d4e13959814a892c6a5e366296ed12ad561 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-s55tr 019792f16922113b95929873c3001d4e13959814a892c6a5e366296ed12ad561 network default NAD default] I0326 23:42:14.029861 4021 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-s55tr, hostIfaceName: 019792f16922113, network: default, NAD default, SandboxID: "019792f16922113b95929873c3001d4e13959814a892c6a5e366296ed12ad561", PCI device ID: , UID: "669e91ef-caf3-41e0-b5ed-c9ebbfee8377", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0326 23:42:14.075516 4021 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-s55tr 019792f16922113b95929873c3001d4e13959814a892c6a5e366296ed12ad561 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-s55tr 019792f16922113b95929873c3001d4e13959814a892c6a5e366296ed12ad561 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:9e:9c:e6:c2:8a\",\"name\":\"019792f16922113\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96cd4d1a-c0c4-4968-8926-1e8668f1f6af\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:42:15.586416 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=736 W0326 23:42:19.531517 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:19.531543 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:19.531551 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:19.531558 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:42:19.556560 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:42:19.556577 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:42:19.556584 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:42:34.164587 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0326 23:42:35.310313 4021 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:35.310859 4021 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:35.328942 4021 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:35.328958 4021 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:35.331806 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0326 23:42:35.331827 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0326 23:42:38.576019 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=83 I0326 23:42:40.662128 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0326 23:42:43.230297 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:42:43.230432 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:42:43.230450 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:42:51.342505 4021 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:42:54.427566 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:42:54.427664 4021 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0326 23:42:54.447601 4021 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 19.964889ms I0326 23:42:54.447624 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 19.992522ms I0326 23:42:54.448890 4021 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 21.342565ms, libovsdb time 868.951µs I0326 23:43:00.794068 4021 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 9693e48c63eb7217a7c1ac98add512f2c50c5b7e2d423e3d0ac5365fec8d60f0 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 9693e48c63eb7217a7c1ac98add512f2c50c5b7e2d423e3d0ac5365fec8d60f0 network default NAD default] I0326 23:43:00.802317 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 9693e48c63eb721, network: default, NAD default, SandboxID: "9693e48c63eb7217a7c1ac98add512f2c50c5b7e2d423e3d0ac5365fec8d60f0", PCI device ID: , UID: "c2975ef2-c1d1-4313-9a3c-a01f6c3f8b1b", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0326 23:43:00.850611 4021 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 9693e48c63eb7217a7c1ac98add512f2c50c5b7e2d423e3d0ac5365fec8d60f0 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 9693e48c63eb7217a7c1ac98add512f2c50c5b7e2d423e3d0ac5365fec8d60f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:b9:51:6b:95:30\",\"name\":\"9693e48c63eb721\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/74146bfb-0d58-41ab-9738-1afca233856f\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:43:01.636519 4021 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:01.637142 4021 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:02.351817 4021 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:02.351845 4021 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:02.358016 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:43:02.358044 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0326 23:43:04.380460 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:04.380484 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:04.382594 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:43:05.841507 4021 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:43:07.373089 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-8jbmw for pod on switch ip-10-0-2-65.ec2.internal I0326 23:43:07.373182 4021 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw I0326 23:43:07.387418 4021 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw] pod update took 14.260999ms I0326 23:43:07.387440 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw] addLogicalPort annotation time took 14.290314ms I0326 23:43:07.388848 4021 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw] addLogicalPort took 15.779518ms, libovsdb time 1.038735ms I0326 23:43:08.333737 4021 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw 00112f0f39aa1fdbebd5e6718d50864202d1b0ca147fe821a9d98ace133bcdb8 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw 00112f0f39aa1fdbebd5e6718d50864202d1b0ca147fe821a9d98ace133bcdb8 network default NAD default] I0326 23:43:08.343522 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-8jbmw, hostIfaceName: 00112f0f39aa1fd, network: default, NAD default, SandboxID: "00112f0f39aa1fdbebd5e6718d50864202d1b0ca147fe821a9d98ace133bcdb8", PCI device ID: , UID: "ea52c144-a6b1-48e8-bb93-6b2b001a07f8", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0326 23:43:08.388603 4021 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw 00112f0f39aa1fdbebd5e6718d50864202d1b0ca147fe821a9d98ace133bcdb8 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-8jbmw 00112f0f39aa1fdbebd5e6718d50864202d1b0ca147fe821a9d98ace133bcdb8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:60:cb:22:1b:28\",\"name\":\"00112f0f39aa1fd\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/abe8d8a0-c480-40ec-b01b-b6e819bad39f\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:43:12.365656 4021 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:43:12.365676 4021 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:43:12.365683 4021 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0326 23:43:24.378407 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-cx9qs] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-cx9qs for pod on switch ip-10-0-2-65.ec2.internal I0326 23:43:24.378539 4021 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-cx9qs I0326 23:43:24.391164 4021 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-cx9qs] pod update took 12.647417ms I0326 23:43:24.391186 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-cx9qs] addLogicalPort annotation time took 12.675172ms I0326 23:43:24.392646 4021 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-cx9qs] addLogicalPort took 14.253203ms, libovsdb time 1.019382ms I0326 23:43:24.701004 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0326 23:43:24.766930 4021 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-cx9qs cbb73378eb9eb63a2ca21862172a76bc421f69d3a47aefb37146421a0607d46e network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-cx9qs cbb73378eb9eb63a2ca21862172a76bc421f69d3a47aefb37146421a0607d46e network default NAD default] I0326 23:43:24.775395 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-cx9qs, hostIfaceName: cbb73378eb9eb63, network: default, NAD default, SandboxID: "cbb73378eb9eb63a2ca21862172a76bc421f69d3a47aefb37146421a0607d46e", PCI device ID: , UID: "72ddda45-48a0-4646-9a31-ab9e56e49e63", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0326 23:43:24.820264 4021 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-cx9qs cbb73378eb9eb63a2ca21862172a76bc421f69d3a47aefb37146421a0607d46e network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-cx9qs cbb73378eb9eb63a2ca21862172a76bc421f69d3a47aefb37146421a0607d46e network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:68:24:63:e9:46\",\"name\":\"cbb73378eb9eb63\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9d96915c-634c-4203-a34f-e5506126fe27\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:43:26.337709 4021 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:43:37.749924 4021 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:37.750428 4021 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:38.544211 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-nzgpx for pod on switch ip-10-0-2-65.ec2.internal I0326 23:43:38.544338 4021 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx I0326 23:43:38.561074 4021 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx] pod update took 16.763725ms I0326 23:43:38.561097 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx] addLogicalPort annotation time took 16.791778ms I0326 23:43:38.562323 4021 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx] addLogicalPort took 18.128722ms, libovsdb time 821.563µs I0326 23:43:38.677425 4021 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-8s9jr for pod on switch ip-10-0-2-65.ec2.internal I0326 23:43:38.677520 4021 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr I0326 23:43:38.700962 4021 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr] pod update took 23.463077ms I0326 23:43:38.700986 4021 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr] addLogicalPort annotation time took 23.494125ms I0326 23:43:38.702284 4021 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr] addLogicalPort took 24.876884ms, libovsdb time 881.709µs I0326 23:43:38.902763 4021 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx 2b34ce9fa64126491f806af6a103263cf80ee1e088389f377a1fbae442b701f3 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx 2b34ce9fa64126491f806af6a103263cf80ee1e088389f377a1fbae442b701f3 network default NAD default] I0326 23:43:38.911112 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-nzgpx, hostIfaceName: 2b34ce9fa641264, network: default, NAD default, SandboxID: "2b34ce9fa64126491f806af6a103263cf80ee1e088389f377a1fbae442b701f3", PCI device ID: , UID: "0db4c121-c9ac-4c25-a7e6-0c191526517c", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0326 23:43:38.960711 4021 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx 2b34ce9fa64126491f806af6a103263cf80ee1e088389f377a1fbae442b701f3 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-nzgpx 2b34ce9fa64126491f806af6a103263cf80ee1e088389f377a1fbae442b701f3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:24:75:a0:fc:da\",\"name\":\"2b34ce9fa641264\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7511111b-55f0-4ef6-b568-fd19c311cf05\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:43:39.039550 4021 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr 122a6de56902b5349725a97e1321a68cfddc2ebbc5be51fba74ca73407d8fc07 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr 122a6de56902b5349725a97e1321a68cfddc2ebbc5be51fba74ca73407d8fc07 network default NAD default] I0326 23:43:39.047853 4021 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-8s9jr, hostIfaceName: 122a6de56902b53, network: default, NAD default, SandboxID: "122a6de56902b5349725a97e1321a68cfddc2ebbc5be51fba74ca73407d8fc07", PCI device ID: , UID: "a1a5da6c-8aad-41d9-8cf1-3f4323464af3", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0326 23:43:39.093616 4021 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr 122a6de56902b5349725a97e1321a68cfddc2ebbc5be51fba74ca73407d8fc07 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8s9jr 122a6de56902b5349725a97e1321a68cfddc2ebbc5be51fba74ca73407d8fc07 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:f9:e6:38:bf:46\",\"name\":\"122a6de56902b53\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e6030db-ba37-43a9-8257-3fa627082eca\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:43:41.989403 4021 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0326 23:43:43.761227 4021 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-vmcfl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:43.761771 4021 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-vmcfl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:44.790726 4021 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-vmcfl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:44.790756 4021 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-vmcfl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:45.143363 4021 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:45.143911 4021 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:45.628010 4021 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:45.628037 4021 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:45.634278 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:43:45.634302 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0326 23:43:47.665414 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:47.665438 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:43:47.665442 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:52.998622 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0326 23:44:02.666582 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0326 23:44:04.577778 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=108 I0326 23:44:37.896958 4021 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:37.896981 4021 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:37.904028 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:44:37.904055 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:44:47.122790 4021 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:44:47.123318 4021 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:44:47.142096 4021 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:47.142117 4021 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:47.145208 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:44:47.145229 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:44:48.751151 4021 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-jbmhq-p4cjm] creating logical port tsf-tas_segment-backup-installation-jbmhq-p4cjm for pod on switch ip-10-0-2-65.ec2.internal I0326 23:44:48.751243 4021 kube.go:256] Updating pod tsf-tas/segment-backup-installation-jbmhq-p4cjm I0326 23:44:48.762450 4021 pod.go:62] [tsf-tas/segment-backup-installation-jbmhq-p4cjm] pod update took 11.229116ms I0326 23:44:48.762480 4021 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-jbmhq-p4cjm] addLogicalPort annotation time took 11.26307ms I0326 23:44:48.763941 4021 pods.go:271] [tsf-tas/segment-backup-installation-jbmhq-p4cjm] addLogicalPort took 12.808891ms, libovsdb time 1.035875ms I0326 23:44:49.113132 4021 cni.go:403] [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default] I0326 23:44:49.121474 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-jbmhq-p4cjm, hostIfaceName: 0a8c3f3b42c9cab, network: default, NAD default, SandboxID: "0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9", PCI device ID: , UID: "11ddae5b-a25d-420b-9879-fddf9ade16ed", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0326 23:44:49.168027 4021 cni.go:424] [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:af:07:14:e9:58\",\"name\":\"0a8c3f3b42c9cab\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f5e60132-f633-4332-b639-2c2029cbe218\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:44:50.448846 4021 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-d8dms-vp26m] creating logical port tsf-tas_rekor-createtree-job-d8dms-vp26m for pod on switch ip-10-0-2-65.ec2.internal I0326 23:44:50.448985 4021 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-d8dms-vp26m I0326 23:44:50.460194 4021 pod.go:62] [tsf-tas/rekor-createtree-job-d8dms-vp26m] pod update took 11.228593ms I0326 23:44:50.460286 4021 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-d8dms-vp26m] addLogicalPort annotation time took 11.324998ms I0326 23:44:50.461852 4021 pods.go:271] [tsf-tas/rekor-createtree-job-d8dms-vp26m] addLogicalPort took 13.028588ms, libovsdb time 1.076255ms I0326 23:44:50.820156 4021 cni.go:403] [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default] I0326 23:44:50.830332 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-d8dms-vp26m, hostIfaceName: 740966c5f3c94aa, network: default, NAD default, SandboxID: "740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180", PCI device ID: , UID: "60fdf4b0-6d90-4b16-af93-8f4a2e44629d", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0326 23:44:50.881545 4021 cni.go:424] [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:01:38:d0:1c:b4\",\"name\":\"740966c5f3c94aa\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/81dc36db-22fc-4251-9fee-1d9cdb212879\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:44:52.033269 4021 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:52.033302 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:57.066692 4021 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0326 23:44:57.920670 4021 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0326 23:44:59.079123 4021 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0326 23:45:01.410719 4021 cni.go:403] [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default] I0326 23:45:01.492786 4021 cni.go:424] [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-jbmhq-p4cjm 0a8c3f3b42c9cab3761ebf0c68e916230624093dfae79896df638a5503451ce9 network default NAD default], result "{}", err I0326 23:45:01.542388 4021 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-jbmhq-p4cjm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:01.542452 4021 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-jbmhq-p4cjm I0326 23:45:01.544378 4021 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-jbmhq-p4cjm, ips: 10.129.0.35 I0326 23:45:01.544413 4021 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-jbmhq-p4cjm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:02.561697 4021 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-jbmhq-p4cjm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:02.561723 4021 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-jbmhq-p4cjm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:23.671227 4021 egressfirewall.go:628] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0326 23:45:23.671275 4021 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0326 23:45:23.671298 4021 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 30.706µs I0326 23:45:37.421973 4021 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:37.422550 4021 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:37.442410 4021 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:37.442430 4021 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:37.445774 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0326 23:45:37.445793 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0326 23:45:44.302468 4021 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-ctr6k-gxz4z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:44.303044 4021 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-ctr6k-gxz4z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:45.184569 4021 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-ctr6k-gxz4z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:45.184595 4021 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-ctr6k-gxz4z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:46.478264 4021 obj_retry.go:464] Detected object tsf/patch-tekton-config-p9frc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:46.478792 4021 obj_retry.go:464] Detected object tsf/patch-tekton-config-p9frc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:47.364747 4021 obj_retry.go:458] Detected object tsf/patch-tekton-config-p9frc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:47.364768 4021 obj_retry.go:458] Detected object tsf/patch-tekton-config-p9frc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:45:48.199993 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0326 23:45:51.661611 4021 cni.go:403] [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default] I0326 23:45:51.751683 4021 cni.go:424] [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-d8dms-vp26m 740966c5f3c94aa3d06af0ed025883d1afa452eb0e78d597f62e5409674d3180 network default NAD default], result "{}", err I0326 23:45:51.824295 4021 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-d8dms-vp26m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:51.824338 4021 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-d8dms-vp26m I0326 23:45:51.825827 4021 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-d8dms-vp26m, ips: 10.129.0.36 I0326 23:45:51.825862 4021 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-d8dms-vp26m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:52.676274 4021 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-d8dms-vp26m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:52.676305 4021 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-d8dms-vp26m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:56.656408 4021 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-b6bbdc55c-cpk85] creating logical port tsf-tas_rekor-redis-b6bbdc55c-cpk85 for pod on switch ip-10-0-2-65.ec2.internal I0326 23:45:56.656510 4021 kube.go:256] Updating pod tsf-tas/rekor-redis-b6bbdc55c-cpk85 I0326 23:45:56.671579 4021 pod.go:62] [tsf-tas/rekor-redis-b6bbdc55c-cpk85] pod update took 15.097037ms I0326 23:45:56.671605 4021 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-b6bbdc55c-cpk85] addLogicalPort annotation time took 15.130195ms I0326 23:45:56.672948 4021 pods.go:271] [tsf-tas/rekor-redis-b6bbdc55c-cpk85] addLogicalPort took 16.558288ms, libovsdb time 933.952µs I0326 23:45:57.390138 4021 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-5b76f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:57.390738 4021 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-5b76f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:57.614208 4021 cni.go:403] [tsf-tas/rekor-redis-b6bbdc55c-cpk85 94daf92b9a17b867adb3a76a50e892bc8a78e9bdfc7fb242ca8b7077386de7a9 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-b6bbdc55c-cpk85 94daf92b9a17b867adb3a76a50e892bc8a78e9bdfc7fb242ca8b7077386de7a9 network default NAD default] I0326 23:45:57.622458 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-b6bbdc55c-cpk85, hostIfaceName: 94daf92b9a17b86, network: default, NAD default, SandboxID: "94daf92b9a17b867adb3a76a50e892bc8a78e9bdfc7fb242ca8b7077386de7a9", PCI device ID: , UID: "24d0201a-0bb4-4558-86bb-a217423f2b5a", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0326 23:45:57.668992 4021 cni.go:424] [tsf-tas/rekor-redis-b6bbdc55c-cpk85 94daf92b9a17b867adb3a76a50e892bc8a78e9bdfc7fb242ca8b7077386de7a9 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-b6bbdc55c-cpk85 94daf92b9a17b867adb3a76a50e892bc8a78e9bdfc7fb242ca8b7077386de7a9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:9f:60:d3:27:d1\",\"name\":\"94daf92b9a17b86\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ecf5927e-3b30-44cc-8f9d-f99562c59fd6\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:45:58.267604 4021 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-5b76f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.267637 4021 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-5b76f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.298732 4021 obj_retry.go:458] Detected object tsf/patch-tekton-config-p9frc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.298756 4021 obj_retry.go:458] Detected object tsf/patch-tekton-config-p9frc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.301579 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-p9frc I0326 23:45:58.301608 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-p9frc W0326 23:45:58.698486 4021 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:45:59.332709 4021 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-5cf49c7696-g25sk] creating logical port tsf-tas_rekor-server-5cf49c7696-g25sk for pod on switch ip-10-0-2-65.ec2.internal I0326 23:45:59.332817 4021 kube.go:256] Updating pod tsf-tas/rekor-server-5cf49c7696-g25sk I0326 23:45:59.344894 4021 pod.go:62] [tsf-tas/rekor-server-5cf49c7696-g25sk] pod update took 12.099542ms I0326 23:45:59.344917 4021 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-5cf49c7696-g25sk] addLogicalPort annotation time took 12.130652ms I0326 23:45:59.346284 4021 pods.go:271] [tsf-tas/rekor-server-5cf49c7696-g25sk] addLogicalPort took 13.593255ms, libovsdb time 858.595µs W0326 23:46:00.270334 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:46:04.493721 4021 cni.go:403] [tsf-tas/rekor-server-5cf49c7696-g25sk fbc4376972861a9a9f30e14a6d27a811813aa8ac30ffdf5008d9fe9d7ebffedd network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-5cf49c7696-g25sk fbc4376972861a9a9f30e14a6d27a811813aa8ac30ffdf5008d9fe9d7ebffedd network default NAD default] I0326 23:46:04.502515 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-5cf49c7696-g25sk, hostIfaceName: fbc4376972861a9, network: default, NAD default, SandboxID: "fbc4376972861a9a9f30e14a6d27a811813aa8ac30ffdf5008d9fe9d7ebffedd", PCI device ID: , UID: "f8b8992b-f18c-4664-b6dd-72598d5deff7", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0326 23:46:04.548445 4021 cni.go:424] [tsf-tas/rekor-server-5cf49c7696-g25sk fbc4376972861a9a9f30e14a6d27a811813aa8ac30ffdf5008d9fe9d7ebffedd network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-5cf49c7696-g25sk fbc4376972861a9a9f30e14a6d27a811813aa8ac30ffdf5008d9fe9d7ebffedd network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:47:b4:f3:32:8a\",\"name\":\"fbc4376972861a9\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2093c119-2dfc-4709-a4d2-304925c2a088\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:46:06.746525 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:46:07.601258 4021 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:07.601871 4021 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:07.620436 4021 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:07.620459 4021 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:07.626613 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0326 23:46:07.626634 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0326 23:46:11.659130 4021 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:11.659690 4021 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:11.678311 4021 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:11.678328 4021 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:11.684728 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0326 23:46:11.684748 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0326 23:46:11.758633 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:46:17.690492 4021 obj_retry.go:464] Detected object tsf-tpa/create-db-mqls9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:17.691081 4021 obj_retry.go:464] Detected object tsf-tpa/create-db-mqls9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:46:17.751772 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:46:18.713281 4021 obj_retry.go:458] Detected object tsf-tpa/create-db-mqls9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:18.713308 4021 obj_retry.go:458] Detected object tsf-tpa/create-db-mqls9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:21.716683 4021 obj_retry.go:464] Detected object tsf-tpa/migrate-db-lrmwt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:21.717220 4021 obj_retry.go:464] Detected object tsf-tpa/migrate-db-lrmwt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:21.843106 4021 obj_retry.go:458] Detected object tsf-tpa/migrate-db-lrmwt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:21.843128 4021 obj_retry.go:458] Detected object tsf-tpa/migrate-db-lrmwt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:24.726717 4021 obj_retry.go:464] Detected object tsf-tpa/create-importers-bm8cq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:24.727251 4021 obj_retry.go:464] Detected object tsf-tpa/create-importers-bm8cq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:24.965154 4021 base_network_controller_pods.go:487] [default/tsf-tpa/server-5fb65d9578-n7hjz] creating logical port tsf-tpa_server-5fb65d9578-n7hjz for pod on switch ip-10-0-2-65.ec2.internal I0326 23:46:24.965249 4021 kube.go:256] Updating pod tsf-tpa/server-5fb65d9578-n7hjz I0326 23:46:24.981116 4021 pod.go:62] [tsf-tpa/server-5fb65d9578-n7hjz] pod update took 15.892378ms I0326 23:46:24.981142 4021 base_network_controller_pods.go:937] [default/tsf-tpa/server-5fb65d9578-n7hjz] addLogicalPort annotation time took 15.922141ms I0326 23:46:24.982391 4021 pods.go:271] [tsf-tpa/server-5fb65d9578-n7hjz] addLogicalPort took 17.255663ms, libovsdb time 846.47µs I0326 23:46:25.015306 4021 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-gdzlt] creating logical port tsf-tpa_importer-74cd48f5c7-gdzlt for pod on switch ip-10-0-2-65.ec2.internal I0326 23:46:25.015415 4021 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-gdzlt I0326 23:46:25.028093 4021 pod.go:62] [tsf-tpa/importer-74cd48f5c7-gdzlt] pod update took 12.703773ms I0326 23:46:25.028116 4021 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-gdzlt] addLogicalPort annotation time took 12.733021ms I0326 23:46:25.029564 4021 pods.go:271] [tsf-tpa/importer-74cd48f5c7-gdzlt] addLogicalPort took 14.271555ms, libovsdb time 942.23µs I0326 23:46:25.749151 4021 obj_retry.go:458] Detected object tsf-tpa/create-importers-bm8cq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:25.749176 4021 obj_retry.go:458] Detected object tsf-tpa/create-importers-bm8cq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:29.577109 4021 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-sl26r-ljq49 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:29.577703 4021 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-sl26r-ljq49 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:30.450714 4021 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-sl26r-ljq49 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:30.450741 4021 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-sl26r-ljq49 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:36.175183 4021 cni.go:403] [tsf-tpa/importer-74cd48f5c7-gdzlt e85dc2801d50152886ac0c7f4369dd666ae96e64dde8c2cac88600a3cd00ced8 network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-gdzlt e85dc2801d50152886ac0c7f4369dd666ae96e64dde8c2cac88600a3cd00ced8 network default NAD default] I0326 23:46:36.183630 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-gdzlt, hostIfaceName: e85dc2801d50152, network: default, NAD default, SandboxID: "e85dc2801d50152886ac0c7f4369dd666ae96e64dde8c2cac88600a3cd00ced8", PCI device ID: , UID: "aa8d96d8-09a9-4a5a-aad9-f2e1ebed62dd", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0326 23:46:36.231689 4021 cni.go:424] [tsf-tpa/importer-74cd48f5c7-gdzlt e85dc2801d50152886ac0c7f4369dd666ae96e64dde8c2cac88600a3cd00ced8 network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-gdzlt e85dc2801d50152886ac0c7f4369dd666ae96e64dde8c2cac88600a3cd00ced8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:99:3d:24:3a:12\",\"name\":\"e85dc2801d50152\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ccfd0acc-af31-4226-ab6e-901476e81c73\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:36.425380 4021 cni.go:403] [tsf-tpa/server-5fb65d9578-n7hjz e80bb6ef13146e8a3a2043534d3c5469f8ce5c1cf9e1897b26f98e29a91b5c03 network default NAD default] ADD starting CNI request [tsf-tpa/server-5fb65d9578-n7hjz e80bb6ef13146e8a3a2043534d3c5469f8ce5c1cf9e1897b26f98e29a91b5c03 network default NAD default] I0326 23:46:36.434599 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-5fb65d9578-n7hjz, hostIfaceName: e80bb6ef13146e8, network: default, NAD default, SandboxID: "e80bb6ef13146e8a3a2043534d3c5469f8ce5c1cf9e1897b26f98e29a91b5c03", PCI device ID: , UID: "9ac10680-e67d-4764-af78-debc16fb4dbd", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0326 23:46:36.480026 4021 cni.go:424] [tsf-tpa/server-5fb65d9578-n7hjz e80bb6ef13146e8a3a2043534d3c5469f8ce5c1cf9e1897b26f98e29a91b5c03 network default NAD default] ADD finished CNI request [tsf-tpa/server-5fb65d9578-n7hjz e80bb6ef13146e8a3a2043534d3c5469f8ce5c1cf9e1897b26f98e29a91b5c03 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:ae:30:2b:5d:8a\",\"name\":\"e80bb6ef13146e8\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/605f1507-14ac-4f22-8354-ce627f152675\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:46:38.916153 4021 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0326 23:46:46.526760 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0326 23:46:49.944662 4021 util.go:959] Failed to build node endpoints for node ip-10-0-2-65.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0326 23:47:05.989374 4021 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:05.989956 4021 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:06.010142 4021 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:06.010161 4021 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:06.017075 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0326 23:47:06.017101 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0326 23:47:06.703766 4021 namespace.go:98] [build-service] adding namespace I0326 23:47:06.704242 4021 namespace.go:98] [image-controller] adding namespace I0326 23:47:06.704790 4021 namespace.go:102] [build-service] adding namespace took 1.004756ms I0326 23:47:06.705051 4021 namespace.go:98] [integration-service] adding namespace I0326 23:47:06.705503 4021 namespace.go:102] [image-controller] adding namespace took 1.246281ms I0326 23:47:06.706165 4021 namespace.go:102] [integration-service] adding namespace took 1.09921ms I0326 23:47:06.744427 4021 namespace.go:142] [build-service] updating namespace I0326 23:47:06.774178 4021 namespace.go:142] [build-service] updating namespace I0326 23:47:06.783396 4021 namespace.go:142] [image-controller] updating namespace I0326 23:47:06.827411 4021 namespace.go:142] [image-controller] updating namespace I0326 23:47:06.828151 4021 namespace.go:142] [integration-service] updating namespace I0326 23:47:06.862414 4021 namespace.go:142] [integration-service] updating namespace I0326 23:47:06.964241 4021 namespace.go:142] [build-service] updating namespace I0326 23:47:06.986992 4021 namespace.go:142] [integration-service] updating namespace I0326 23:47:07.032253 4021 namespace.go:98] [release-service] adding namespace I0326 23:47:07.033038 4021 namespace.go:102] [release-service] adding namespace took 760.865µs I0326 23:47:07.098809 4021 namespace.go:142] [release-service] updating namespace I0326 23:47:07.117286 4021 namespace.go:142] [konflux-ui] updating namespace I0326 23:47:07.122245 4021 namespace.go:98] [konflux-info] adding namespace I0326 23:47:07.122990 4021 namespace.go:102] [konflux-info] adding namespace took 731.466µs I0326 23:47:07.123788 4021 namespace.go:142] [release-service] updating namespace I0326 23:47:07.168475 4021 namespace.go:98] [namespace-lister] adding namespace I0326 23:47:07.169375 4021 namespace.go:102] [namespace-lister] adding namespace took 876.642µs I0326 23:47:07.182672 4021 namespace.go:142] [konflux-info] updating namespace I0326 23:47:07.194402 4021 namespace.go:98] [enterprise-contract-service] adding namespace I0326 23:47:07.195120 4021 namespace.go:102] [enterprise-contract-service] adding namespace took 703.586µs I0326 23:47:07.203381 4021 namespace.go:142] [image-controller] updating namespace I0326 23:47:07.290931 4021 namespace.go:142] [konflux-info] updating namespace I0326 23:47:07.290981 4021 namespace.go:142] [namespace-lister] updating namespace I0326 23:47:07.317582 4021 namespace.go:98] [default-tenant] adding namespace I0326 23:47:07.318509 4021 namespace.go:102] [default-tenant] adding namespace took 908.623µs I0326 23:47:07.370574 4021 namespace.go:98] [segment-bridge] adding namespace I0326 23:47:07.371422 4021 namespace.go:102] [segment-bridge] adding namespace took 823.447µs I0326 23:47:07.492675 4021 namespace.go:142] [namespace-lister] updating namespace I0326 23:47:07.690280 4021 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:47:07.829825 4021 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0326 23:47:07.830506 4021 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0326 23:47:07.830524 4021 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0326 23:47:07.836255 4021 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0326 23:47:07.872370 4021 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:47:07.881203 4021 namespace.go:142] [default-tenant] updating namespace I0326 23:47:08.022301 4021 namespace.go:142] [default-tenant] updating namespace I0326 23:47:08.022388 4021 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0326 23:47:08.022413 4021 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0326 23:47:08.023372 4021 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0326 23:47:08.113447 4021 namespace.go:142] [segment-bridge] updating namespace I0326 23:47:08.183419 4021 namespace.go:142] [segment-bridge] updating namespace I0326 23:47:09.060608 4021 namespace.go:142] [build-service] updating namespace I0326 23:47:09.298203 4021 namespace.go:142] [default-tenant] updating namespace I0326 23:47:09.597286 4021 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:47:09.778737 4021 namespace.go:142] [image-controller] updating namespace I0326 23:47:10.037332 4021 namespace.go:142] [integration-service] updating namespace I0326 23:47:10.251176 4021 namespace.go:142] [konflux-info] updating namespace I0326 23:47:10.489271 4021 namespace.go:142] [namespace-lister] updating namespace I0326 23:47:10.717355 4021 namespace.go:142] [release-service] updating namespace W0326 23:47:10.879890 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0326 23:47:10.929503 4021 namespace.go:142] [segment-bridge] updating namespace I0326 23:47:11.013568 4021 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-8x958 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:11.014248 4021 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-8x958 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:11.052381 4021 namespace.go:142] [build-service] updating namespace I0326 23:47:11.067240 4021 namespace.go:142] [default-tenant] updating namespace I0326 23:47:11.084105 4021 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:47:11.092497 4021 namespace.go:142] [image-controller] updating namespace I0326 23:47:11.105307 4021 namespace.go:142] [integration-service] updating namespace I0326 23:47:11.131203 4021 namespace.go:142] [konflux-info] updating namespace I0326 23:47:11.153421 4021 namespace.go:142] [namespace-lister] updating namespace I0326 23:47:11.161029 4021 namespace.go:142] [release-service] updating namespace I0326 23:47:11.173954 4021 namespace.go:142] [segment-bridge] updating namespace I0326 23:47:11.353166 4021 namespace.go:142] [build-service] updating namespace I0326 23:47:11.457331 4021 namespace.go:142] [default-tenant] updating namespace I0326 23:47:11.555756 4021 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:47:11.657385 4021 namespace.go:142] [image-controller] updating namespace I0326 23:47:11.754987 4021 namespace.go:142] [integration-service] updating namespace I0326 23:47:11.856186 4021 namespace.go:142] [konflux-info] updating namespace I0326 23:47:11.957430 4021 namespace.go:142] [namespace-lister] updating namespace I0326 23:47:12.052301 4021 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-8x958 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:12.052326 4021 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-8x958 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:12.056289 4021 namespace.go:142] [release-service] updating namespace I0326 23:47:12.176337 4021 namespace.go:142] [segment-bridge] updating namespace W0326 23:47:13.899976 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0326 23:47:13.929909 4021 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0326 23:47:15.703282 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 W0326 23:47:19.948053 4021 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0326 23:47:20.581206 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 W0326 23:47:24.987324 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:47:24.991636 4021 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:47:25.021642 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:25.021657 4021 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:25.050613 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:47:25.053560 4021 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:47:25.107283 4021 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:47:25.108553 4021 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:47:25.760766 4021 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:25.760788 4021 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0326 23:47:36.414605 4021 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-h969m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:36.415234 4021 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-h969m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:36.692055 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0326 23:47:37.084449 4021 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-h969m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:37.084473 4021 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-h969m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:37.092398 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-h969m I0326 23:47:37.092427 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-h969m I0326 23:47:47.686391 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0326 23:47:57.581099 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=168 I0326 23:48:32.371833 4021 obj_retry.go:464] Detected object konflux-ui/proxy-7cb54d888d-tkqnz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:32.372435 4021 obj_retry.go:464] Detected object konflux-ui/proxy-7cb54d888d-tkqnz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:32.683028 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0326 23:48:32.715302 4021 obj_retry.go:458] Detected object konflux-ui/proxy-7cb54d888d-tkqnz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:32.715329 4021 obj_retry.go:458] Detected object konflux-ui/proxy-7cb54d888d-tkqnz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:32.721733 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-7cb54d888d-tkqnz I0326 23:48:32.721753 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-7cb54d888d-tkqnz I0326 23:48:34.666283 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0326 23:48:43.327779 4021 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:43.328373 4021 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:43.365244 4021 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:43.365266 4021 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:43.372442 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0326 23:48:43.372469 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0326 23:48:46.675619 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0326 23:48:53.579116 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=55 I0326 23:49:08.726886 4021 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default] I0326 23:49:08.819999 4021 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 2ff0f62f7f7029b9db9c8eb46eabaebf017e0305e7237e8f3e473492d0c1a917 network default NAD default], result "{}", err I0326 23:49:08.886037 4021 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:08.886071 4021 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 I0326 23:49:08.887522 4021 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6, ips: 10.129.0.24 I0326 23:49:08.887556 4021 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:09.695725 4021 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:49:09.695752 4021 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:49:09.703153 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 I0326 23:49:09.703180 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-wghj6 I0326 23:49:21.990642 4021 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-4bj5g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:21.991239 4021 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-4bj5g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:22.723130 4021 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-4bj5g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:49:22.723152 4021 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-4bj5g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:49:22.728824 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-4bj5g I0326 23:49:22.728839 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-4bj5g I0326 23:49:32.581295 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=82 I0326 23:49:37.665922 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0326 23:49:38.704299 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:49:43.589501 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=497 I0326 23:50:04.133933 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-69nkw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.134522 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-69nkw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.997623 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-69nkw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:04.997648 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-69nkw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:17.698318 4021 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0326 23:51:43.581421 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=67 I0326 23:53:09.584558 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0326 23:53:10.584440 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=16 I0326 23:53:40.706982 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0326 23:53:46.669465 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0326 23:54:07.686039 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0326 23:54:11.678857 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0326 23:54:52.706888 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0326 23:54:56.593008 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=15 I0326 23:55:09.584397 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0326 23:55:42.689807 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0326 23:56:09.669675 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0326 23:56:18.046230 4021 obj_retry.go:458] Detected object tsf-tpa/create-db-mqls9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:18.046283 4021 obj_retry.go:458] Detected object tsf-tpa/create-db-mqls9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:18.050802 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-mqls9 I0326 23:56:18.050836 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-mqls9 I0326 23:56:21.026099 4021 obj_retry.go:458] Detected object tsf-tpa/migrate-db-lrmwt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:21.026141 4021 obj_retry.go:458] Detected object tsf-tpa/migrate-db-lrmwt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:21.029627 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-lrmwt I0326 23:56:21.029651 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-lrmwt I0326 23:56:25.026358 4021 obj_retry.go:458] Detected object tsf-tpa/create-importers-bm8cq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:25.026409 4021 obj_retry.go:458] Detected object tsf-tpa/create-importers-bm8cq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:25.028573 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-bm8cq I0326 23:56:25.028601 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-bm8cq W0326 23:56:31.983043 4021 duration.go:461] Config duration recorder: measurement expired for pod/openshift-image-registry/image-registry-66c8fd6f5d-kfmps I0326 23:56:34.697117 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0326 23:58:17.582506 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=36 I0326 23:58:41.710027 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0326 23:58:50.668806 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0326 23:59:49.702289 4021 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0326 23:59:55.712910 4021 namespace.go:98] [default-tenant-managed] adding namespace I0326 23:59:55.713774 4021 namespace.go:102] [default-tenant-managed] adding namespace took 837.202µs I0326 23:59:55.741656 4021 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:59:55.762201 4021 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:59:56.729797 4021 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:59:56.857164 4021 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:59:56.961313 4021 namespace.go:142] [default-tenant-managed] updating namespace I0327 00:00:00.300028 4021 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-nightly-metrics-29576160-khvkc] creating logical port tsf-tas_segment-backup-nightly-metrics-29576160-khvkc for pod on switch ip-10-0-2-65.ec2.internal I0327 00:00:00.300149 4021 kube.go:256] Updating pod tsf-tas/segment-backup-nightly-metrics-29576160-khvkc I0327 00:00:00.315330 4021 pod.go:62] [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc] pod update took 15.202905ms I0327 00:00:00.315377 4021 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-nightly-metrics-29576160-khvkc] addLogicalPort annotation time took 15.256501ms I0327 00:00:00.316534 4021 pods.go:271] [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc] addLogicalPort took 16.52569ms, libovsdb time 754.619µs I0327 00:00:00.587596 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0327 00:00:00.683958 4021 cni.go:403] [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default] I0327 00:00:00.692799 4021 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-nightly-metrics-29576160-khvkc, hostIfaceName: 74c058f42c675d0, network: default, NAD default, SandboxID: "74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661", PCI device ID: , UID: "9beaec53-1046-4fa0-9e55-05f36bdf07c5", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0327 00:00:00.743385 4021 cni.go:424] [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:4f:6f:96:71:01\",\"name\":\"74c058f42c675d0\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d9a20db0-b124-47b6-8691-b1a12afd9023\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 00:00:02.602189 4021 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29576160-mhrlz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:02.602821 4021 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29576160-mhrlz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.070752 4021 cni.go:403] [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default] I0327 00:00:03.156913 4021 cni.go:424] [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-nightly-metrics-29576160-khvkc 74c058f42c675d0349463c23b1c485172f20e8655409bfc7af324796ea45b661 network default NAD default], result "{}", err I0327 00:00:03.213522 4021 obj_retry.go:464] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-khvkc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.213560 4021 pods.go:176] Deleting pod: tsf-tas/segment-backup-nightly-metrics-29576160-khvkc I0327 00:00:03.214981 4021 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-nightly-metrics-29576160-khvkc, ips: 10.129.0.41 I0327 00:00:03.215010 4021 obj_retry.go:464] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-khvkc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.323979 4021 obj_retry.go:458] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-khvkc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:03.324001 4021 obj_retry.go:458] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-khvkc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:03.600961 4021 obj_retry.go:464] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-mzpg4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.601544 4021 obj_retry.go:464] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-mzpg4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.625022 4021 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29576160-mhrlz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:03.625050 4021 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29576160-mhrlz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.486643 4021 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29576160-q5szh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.487081 4021 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29576160-q5szh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.620833 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-m9zmv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.621443 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-m9zmv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.622289 4021 obj_retry.go:458] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-mzpg4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.622310 4021 obj_retry.go:458] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-mzpg4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:05.489782 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-m9zmv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:05.489804 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-m9zmv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:05.512270 4021 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29576160-q5szh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:05.512292 4021 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29576160-q5szh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:45.235711 4021 obj_retry.go:464] Detected object tsf-tas/backfill-redis-29576160-wthnl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:45.236297 4021 obj_retry.go:464] Detected object tsf-tas/backfill-redis-29576160-wthnl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:46.262205 4021 obj_retry.go:458] Detected object tsf-tas/backfill-redis-29576160-wthnl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:46.262228 4021 obj_retry.go:458] Detected object tsf-tas/backfill-redis-29576160-wthnl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:49.693178 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0327 00:00:51.681522 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0327 00:00:52.584264 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0327 00:01:17.086516 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:01:17.087093 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:01:25.587720 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=13 I0327 00:01:32.709803 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0327 00:01:47.323334 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:01:47.323949 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:05.496051 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:05.496618 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:05.673366 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 00:03:07.700707 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0327 00:03:10.690853 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 00:03:40.672412 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0327 00:04:27.596453 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=148 I0327 00:04:34.712748 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0327 00:04:43.817517 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:43.818129 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:46.587097 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0327 00:05:26.022080 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:26.022677 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:28.114578 4021 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod] creating logical port default-tenant_tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod for pod on switch ip-10-0-2-65.ec2.internal I0327 00:05:28.114734 4021 kube.go:256] Updating pod default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod I0327 00:05:28.136978 4021 pod.go:62] [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod] pod update took 22.280436ms I0327 00:05:28.137003 4021 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod] addLogicalPort annotation time took 22.311515ms I0327 00:05:28.138275 4021 pods.go:271] [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod] addLogicalPort took 23.719588ms, libovsdb time 853.691µs I0327 00:05:28.482411 4021 cni.go:403] [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default] I0327 00:05:28.491428 4021 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod, hostIfaceName: 2d8511be8596f8d, network: default, NAD default, SandboxID: "2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c", PCI device ID: , UID: "f7c900bd-5347-41f9-9dda-0b00b79f19dd", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0327 00:05:28.752118 4021 cni.go:424] [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:f8:a2:68:03:e1\",\"name\":\"2d8511be8596f8d\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/802d8642-c0fd-4ffc-9785-cf35dd8cb588\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 00:05:39.591143 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0327 00:06:09.172306 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:09.172864 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:17.840716 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:17.841242 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:19.749262 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:19.749801 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:21.617194 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:21.617770 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:28.556880 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:28.557589 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:31.586113 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=42 W0327 00:06:31.983168 4021 duration.go:461] Config duration recorder: measurement expired for pod/tsf-tas/tuf-repository-init-sl26r-ljq49 I0327 00:06:33.166086 4021 cni.go:403] [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default] I0327 00:06:33.250444 4021 cni.go:424] [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod 2d8511be8596f8d7254e3102f9d116c61a53b279f7e6578aa91d1daab3a1561c network default NAD default], result "{}", err I0327 00:06:33.376967 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:33.377005 4021 pods.go:176] Deleting pod: default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod I0327 00:06:33.378556 4021 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod, ips: 10.129.0.42 I0327 00:06:33.378602 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:33.613808 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:33.614330 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:38.923336 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:38.923878 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:52.774141 4021 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-crrwd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:52.774737 4021 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-crrwd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:53.654209 4021 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-crrwd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:06:53.654239 4021 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-crrwd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:06:53.660483 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-crrwd I0327 00:06:53.660507 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-crrwd I0327 00:07:09.696058 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 W0327 00:07:11.983240 4021 duration.go:461] Config duration recorder: measurement expired for pod/konflux-ui/tsf-konflux-public-key-8x958 I0327 00:07:13.254668 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:07:13.255331 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:07:20.128293 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:07:20.128947 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:07:33.587513 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0327 00:07:35.672081 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0327 00:07:40.705680 4021 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 00:08:05.713310 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0327 00:08:50.494010 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:08:50.494668 4021 obj_retry.go:464] Detected object default-tenant/tsf-comp-skua-on-push-7hfvf-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:09:08.684883 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0327 00:09:36.676411 4021 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 00:09:43.244814 4021 obj_retry.go:464] Detected object default-tenant/my-integration-test-xeyl-vvtz5-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:09:43.245358 4021 obj_retry.go:464] Detected object default-tenant/my-integration-test-xeyl-vvtz5-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:09:53.589897 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0327 00:10:03.966928 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-tfkzc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:03.967470 4021 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-tfkzc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:04.703516 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0327 00:10:04.821544 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-tfkzc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:04.821569 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-tfkzc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:04.850318 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:04.850339 4021 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:04.855413 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 I0327 00:10:04.855442 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-b5ww2 I0327 00:10:12.457822 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:12.458391 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:36.737409 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:36.737942 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:45.824924 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:45.825484 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:46.840565 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:46.841122 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:02.022869 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:02.023535 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.590569 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=13 I0327 00:11:13.111426 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:13.112102 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:37.366058 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:37.366653 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:12:00.607184 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:12:00.607778 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:12:28.599362 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=307 I0327 00:12:28.675679 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0327 00:12:38.694185 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0327 00:12:45.223276 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-verify-conforma-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:12:45.224017 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-verify-conforma-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:12:58.590484 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0327 00:13:00.589104 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=48 I0327 00:13:12.386689 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-push-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:13:12.387336 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-push-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:13:19.742753 4021 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-9831ce8d32-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:13:19.743436 4021 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-9831ce8d32-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:13:20.301419 4021 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-9831ce8d32-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:13:20.301444 4021 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-9831ce8d32-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:13:20.308002 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-9831ce8d32-0 I0327 00:13:20.308058 4021 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-9831ce8d32-0 I0327 00:13:20.476266 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-update-cr-status-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:13:20.476885 4021 obj_retry.go:464] Detected object default-tenant-managed/managed-4zt7x-update-cr-status-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:13:24.716485 4021 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0327 00:13:44.593921 4021 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9