+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-144.ec2.internal ++ [[ -n ip-10-0-1-144.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-144.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 15:09:17.389751335 - disable conntrack on geneve port' I0327 15:09:17.389751335 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0327 15:09:17.406777853 - starting ovnkube-node + echo 'I0327 15:09:17.406777853 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-144.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-144.ec2.internal --init-node ip-10-0-1-144.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-144.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 15:09:17.443782 3959 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 15:09:17.443830 3959 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-bccd260e05.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 15:09:17.445220 3959 kube.go:419] Waiting for certificate I0327 15:09:17.445247 3959 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 15:09:17.445278 3959 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 15:09:17.446336 3959 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:09:17.477029 3959 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:17.477049 3959 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:17.479839 3959 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:17.479919 3959 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-t4zkt" I0327 15:09:17.486730 3959 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-t4zkt" I0327 15:09:17.486796 3959 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:18.446004 3959 kube.go:426] Certificate found I0327 15:09:18.446584 3959 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:09:18.447846 3959 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 15:09:18.490113 3959 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 15:04:17 +0000 UTC" deadline="2026-03-28 10:32:05.192797097 +0000 UTC" I0327 15:09:18.490152 3959 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h22m46.702657353s" I0327 15:09:18.513570 3959 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:09:18.526177 3959 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 15:09:18.526267 3959 factory.go:531] Starting watch factory I0327 15:09:18.526374 3959 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526385 3959 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526395 3959 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526399 3959 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526401 3959 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526407 3959 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526418 3959 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526422 3959 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526425 3959 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526405 3959 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526737 3959 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.526760 3959 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.527154 3959 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 15:09:18.534963 3959 controller_manager.go:368] Starting the ovnkube controller I0327 15:09:18.534978 3959 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-144.ec2.internal I0327 15:09:18.535047 3959 controller_manager.go:393] NBDB zone sync took: 57.174µs I0327 15:09:18.535061 3959 factory.go:531] Starting watch factory I0327 15:09:18.547948 3959 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.550304 3959 ovn_northd.go:121] Registering OVN northd metrics I0327 15:09:18.550327 3959 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 15:09:18.551587 3959 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.551997 3959 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.552856 3959 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.553780 3959 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.553883 3959 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 15:09:18.553931 3959 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 15:09:18.557514 3959 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:18.570453 3959 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 15:09:18.631595 3959 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:09:18.631631 3959 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:09:18.631635 3959 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:09:18.631641 3959 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:09:18.631645 3959 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:09:18.631650 3959 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:09:18.631782 3959 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:18.631808 3959 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:18.631784 3959 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:18.631879 3959 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:18.634625 3959 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:18.634627 3959 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:18.637726 3959 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:09:18.637739 3959 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:09:18.637826 3959 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.637838 3959 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.640095 3959 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.640202 3959 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:09:18.640211 3959 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:09:18.640215 3959 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:09:18.640218 3959 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:09:18.640221 3959 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:09:18.640224 3959 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:09:18.643821 3959 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:09:18.643905 3959 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.643946 3959 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.645354 3959 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:09:18.645365 3959 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:09:18.646199 3959 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.648991 3959 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:09:18.649045 3959 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.649057 3959 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.650420 3959 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:09:18.651310 3959 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.655052 3959 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:09:18.655201 3959 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.655217 3959 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.655559 3959 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:09:18.657367 3959 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.661082 3959 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:09:18.661094 3959 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:09:18.661219 3959 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.661237 3959 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.663573 3959 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.666780 3959 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:09:18.666854 3959 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:09:18.667554 3959 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:09:18.667572 3959 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:09:18.669857 3959 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:09:18.672716 3959 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:09:18.672725 3959 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:09:18.672803 3959 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.672817 3959 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.672818 3959 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.672835 3959 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.675287 3959 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.675752 3959 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:18.678565 3959 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:09:18.678577 3959 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:09:18.678584 3959 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:09:18.678601 3959 factory.go:636] Watch Factory start up complete, took: 152.331608ms I0327 15:09:18.678687 3959 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 15:09:18.684126 3959 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:09:18.684137 3959 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:09:18.684144 3959 factory.go:636] Watch Factory start up complete, took: 149.084202ms I0327 15:09:18.684152 3959 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-144.ec2.internal" zone I0327 15:09:18.715388 3959 default_node_network_controller.go:709] Initializing the default node network controller I0327 15:09:18.763159 3959 udn_isolation.go:101] Starting UDN host isolation manager I0327 15:09:18.769797 3959 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 15:09:18.823966 3959 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 15:09:18.824058 3959 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 15:09:18.824077 3959 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 15:09:18.860140 3959 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 15:09:18.860215 3959 default_node_network_controller.go:808] Node ip-10-0-1-144.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0327 15:09:18.865459 3959 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.144"] k8s.ovn.org/zone-name:ip-10-0-1-144.ec2.internal] on node ip-10-0-1-144.ec2.internal I0327 15:09:18.884820 3959 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 15:09:18.890188 3959 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 15:09:18.890316 3959 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0327 15:09:18.890506 3959 gateway_init.go:263] Preparing Gateway I0327 15:09:18.890518 3959 gateway_shared_intf.go:1658] Creating new gateway I0327 15:09:18.934648 3959 controller_manager.go:427] Waiting for node in zone sync took: 250.477256ms I0327 15:09:18.939592 3959 controller_manager.go:608] Switching to transit router for layer2 networks I0327 15:09:18.939606 3959 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-144.ec2.internal I0327 15:09:18.955438 3959 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:09:18.960763 3959 controller_manager.go:288] SCTP support detected in OVN I0327 15:09:18.961350 3959 services_controller.go:69] Creating services controller for network=default I0327 15:09:18.961452 3959 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 15:09:18.961555 3959 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 15:09:18.961590 3959 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 15:09:18.961611 3959 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 15:09:18.961628 3959 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:09:18.961849 3959 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 15:09:18.961908 3959 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 15:09:18.961939 3959 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 57.87µs I0327 15:09:18.961942 3959 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 24.16µs I0327 15:09:18.961963 3959 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 56.748µs I0327 15:09:18.961971 3959 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 15:09:18.961979 3959 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 15:09:18.961987 3959 default_network_controller.go:321] Starting the default network controller I0327 15:09:18.962037 3959 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 9.006µs I0327 15:09:18.962084 3959 udn_enabled_svc.go:95] Performing full resync I0327 15:09:18.962332 3959 udn_enabled_svc.go:99] Waiting for handler to sync I0327 15:09:18.962345 3959 udn_enabled_svc.go:104] Starting worker I0327 15:09:18.965105 3959 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:09:18.970218 3959 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 15:09:18.970281 3959 repair.go:33] Syncing exgw routes took 51.049µs I0327 15:09:18.970301 3959 default_network_controller.go:407] Starting all the Watchers... I0327 15:09:18.970430 3959 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 15:09:18.970432 3959 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 15:09:18.970431 3959 namespace.go:98] [openshift-authentication] adding namespace I0327 15:09:18.970444 3959 namespace.go:98] [kube-system] adding namespace I0327 15:09:18.970450 3959 namespace.go:98] [openshift-etcd] adding namespace I0327 15:09:18.970455 3959 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 15:09:18.970464 3959 namespace.go:98] [openshift-multus] adding namespace I0327 15:09:18.970472 3959 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 15:09:18.970507 3959 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 15:09:18.970515 3959 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 15:09:18.970518 3959 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 15:09:18.970525 3959 namespace.go:98] [openshift-network-operator] adding namespace I0327 15:09:18.970531 3959 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 15:09:18.970558 3959 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 15:09:18.970570 3959 namespace.go:98] [openshift-console] adding namespace I0327 15:09:18.971062 3959 namespace.go:102] [openshift-backplane-cee] adding namespace took 621.456µs I0327 15:09:18.971077 3959 namespace.go:98] [openshift-machine-api] adding namespace I0327 15:09:18.971114 3959 iptables.go:108] Creating table: filter chain: OUTPUT I0327 15:09:18.971492 3959 namespace.go:102] [openshift-machine-api] adding namespace took 410.797µs I0327 15:09:18.971505 3959 namespace.go:98] [kube-node-lease] adding namespace I0327 15:09:18.972146 3959 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 1.70085ms I0327 15:09:18.972162 3959 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 15:09:18.972697 3959 namespace.go:102] [openshift-authentication] adding namespace took 2.248698ms I0327 15:09:18.972715 3959 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 15:09:18.973186 3959 namespace.go:102] [openshift-kube-scheduler] adding namespace took 464.05µs I0327 15:09:18.973199 3959 namespace.go:98] [openshift-controller-manager] adding namespace W0327 15:09:18.973202 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-8qmwh: pod kube-system/global-pull-secret-syncer-8qmwh: no pod IPs found W0327 15:09:18.973224 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-4cpwh: pod kube-system/global-pull-secret-syncer-4cpwh: no pod IPs found I0327 15:09:18.973819 3959 namespace.go:102] [kube-system] adding namespace took 3.363137ms I0327 15:09:18.973839 3959 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 15:09:18.974442 3959 namespace.go:102] [openshift-etcd] adding namespace took 3.968281ms I0327 15:09:18.974453 3959 namespace.go:98] [openshift-ingress-operator] adding namespace W0327 15:09:18.975027 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-64l54: pod openshift-multus/network-metrics-daemon-64l54: no pod IPs found W0327 15:09:18.975040 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-dr7nn: pod openshift-multus/network-metrics-daemon-dr7nn: no pod IPs found W0327 15:09:18.975051 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-nfpdk: pod openshift-multus/network-metrics-daemon-nfpdk: no pod IPs found I0327 15:09:18.975070 3959 namespace.go:102] [openshift-operators-redhat] adding namespace took 4.59219ms I0327 15:09:18.975078 3959 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 15:09:18.975702 3959 namespace.go:102] [openshift-multus] adding namespace took 5.216131ms I0327 15:09:18.975737 3959 namespace.go:98] [openshift-package-operator] adding namespace I0327 15:09:18.976249 3959 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.760771ms I0327 15:09:18.976265 3959 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 15:09:18.977123 3959 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.604268ms I0327 15:09:18.977149 3959 namespace.go:98] [openshift-logging] adding namespace I0327 15:09:18.977239 3959 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 15:09:18.977252 3959 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-144.ec2.internal","mac-address":"0a:ff:d3:61:56:43","ip-addresses":["10.0.1.144/24"],"ip-address":"10.0.1.144/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:15418104-dcbb-4f24-b50d-233cff74ddc6 k8s.ovn.org/node-encap-ips:["10.0.1.144"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.144/24"} k8s.ovn.org/zone-name:ip-10-0-1-144.ec2.internal] on node ip-10-0-1-144.ec2.internal I0327 15:09:18.977603 3959 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.079409ms I0327 15:09:18.977617 3959 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 15:09:18.978069 3959 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.546168ms I0327 15:09:18.978089 3959 namespace.go:98] [openshift-cluster-version] adding namespace I0327 15:09:18.978539 3959 namespace.go:102] [openshift-network-operator] adding namespace took 8.009565ms I0327 15:09:18.978551 3959 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 15:09:18.979057 3959 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.497902ms I0327 15:09:18.979074 3959 namespace.go:98] [openshift-node] adding namespace I0327 15:09:18.979466 3959 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.901139ms I0327 15:09:18.979478 3959 namespace.go:98] [openshift-config] adding namespace I0327 15:09:18.979892 3959 namespace.go:102] [openshift-console] adding namespace took 9.311986ms I0327 15:09:18.979906 3959 namespace.go:98] [openshift-insights] adding namespace I0327 15:09:18.980317 3959 namespace.go:102] [kube-node-lease] adding namespace took 8.807032ms I0327 15:09:18.980333 3959 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 15:09:18.980683 3959 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.516309ms I0327 15:09:18.980698 3959 namespace.go:98] [openshift-ingress] adding namespace I0327 15:09:18.981081 3959 namespace.go:102] [openshift-controller-manager] adding namespace took 7.877776ms I0327 15:09:18.981092 3959 namespace.go:98] [dedicated-admin] adding namespace I0327 15:09:18.981459 3959 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.614055ms I0327 15:09:18.981472 3959 namespace.go:98] [openshift-dns-operator] adding namespace I0327 15:09:18.981983 3959 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.5257ms I0327 15:09:18.981994 3959 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 15:09:18.982364 3959 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.281358ms I0327 15:09:18.982379 3959 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 15:09:18.982757 3959 namespace.go:102] [openshift-package-operator] adding namespace took 7.013617ms I0327 15:09:18.982771 3959 namespace.go:98] [kube-public] adding namespace I0327 15:09:18.983118 3959 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.847785ms I0327 15:09:18.983131 3959 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 15:09:18.983506 3959 namespace.go:102] [openshift-logging] adding namespace took 6.351479ms I0327 15:09:18.983519 3959 namespace.go:98] [openshift-host-network] adding namespace I0327 15:09:18.983903 3959 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.281909ms I0327 15:09:18.983915 3959 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 15:09:18.984288 3959 namespace.go:102] [openshift-cluster-version] adding namespace took 6.191819ms I0327 15:09:18.984301 3959 namespace.go:98] [openshift-marketplace] adding namespace I0327 15:09:18.984649 3959 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.093378ms I0327 15:09:18.984676 3959 namespace.go:98] [openshift-operators] adding namespace I0327 15:09:18.985027 3959 namespace.go:102] [openshift-node] adding namespace took 5.948264ms I0327 15:09:18.985037 3959 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 15:09:18.985412 3959 namespace.go:102] [openshift-config] adding namespace took 5.928328ms I0327 15:09:18.985424 3959 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 15:09:18.985817 3959 namespace.go:102] [openshift-insights] adding namespace took 5.906804ms I0327 15:09:18.985828 3959 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 15:09:18.986193 3959 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.854158ms I0327 15:09:18.986205 3959 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 15:09:18.986547 3959 namespace.go:102] [openshift-ingress] adding namespace took 5.844131ms I0327 15:09:18.986558 3959 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 15:09:18.986950 3959 namespace.go:102] [dedicated-admin] adding namespace took 5.8527ms I0327 15:09:18.986962 3959 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 15:09:18.987309 3959 namespace.go:102] [openshift-dns-operator] adding namespace took 5.832581ms I0327 15:09:18.987319 3959 namespace.go:98] [openshift-dns] adding namespace I0327 15:09:18.987679 3959 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.681679ms I0327 15:09:18.987688 3959 namespace.go:98] [openshift-config-operator] adding namespace I0327 15:09:18.988090 3959 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.704737ms I0327 15:09:18.988104 3959 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 15:09:18.988448 3959 namespace.go:102] [kube-public] adding namespace took 5.672383ms I0327 15:09:18.988460 3959 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 15:09:18.988869 3959 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.734054ms I0327 15:09:18.988879 3959 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 15:09:18.989401 3959 namespace.go:102] [openshift-host-network] adding namespace took 5.878124ms I0327 15:09:18.989409 3959 namespace.go:98] [openshift-config-managed] adding namespace I0327 15:09:18.989944 3959 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.021672ms I0327 15:09:18.989960 3959 namespace.go:98] [openshift-network-console] adding namespace I0327 15:09:18.990300 3959 namespace.go:102] [openshift-marketplace] adding namespace took 5.992742ms I0327 15:09:18.990314 3959 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 15:09:18.990684 3959 namespace.go:102] [openshift-operators] adding namespace took 6.003441ms I0327 15:09:18.990698 3959 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0327 15:09:18.991109 3959 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.067428ms I0327 15:09:18.991120 3959 namespace.go:98] [openshift-infra] adding namespace I0327 15:09:18.991468 3959 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.038976ms I0327 15:09:18.991480 3959 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace W0327 15:09:18.991819 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-dcfk8: pod openshift-network-diagnostics/network-check-target-dcfk8: no pod IPs found W0327 15:09:18.991831 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rwdfv: pod openshift-network-diagnostics/network-check-target-rwdfv: no pod IPs found W0327 15:09:18.991838 3959 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-xrnlc: pod openshift-network-diagnostics/network-check-target-xrnlc: no pod IPs found I0327 15:09:18.991863 3959 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.031025ms I0327 15:09:18.991872 3959 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 15:09:18.992308 3959 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.098516ms I0327 15:09:18.992681 3959 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.118695ms I0327 15:09:18.992697 3959 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 15:09:18.993137 3959 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.170243ms I0327 15:09:18.993152 3959 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 15:09:18.993983 3959 namespace.go:102] [openshift-dns] adding namespace took 6.657582ms I0327 15:09:18.993998 3959 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 15:09:18.994602 3959 namespace.go:102] [openshift-config-operator] adding namespace took 6.904857ms I0327 15:09:18.994622 3959 namespace.go:98] [openshift-image-registry] adding namespace I0327 15:09:18.995175 3959 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.064337ms I0327 15:09:18.995188 3959 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 15:09:18.995572 3959 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.105032ms I0327 15:09:18.995587 3959 namespace.go:98] [openshift-backplane] adding namespace I0327 15:09:18.996130 3959 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.243535ms I0327 15:09:18.996147 3959 namespace.go:98] [openshift-console-operator] adding namespace I0327 15:09:18.996585 3959 namespace.go:102] [openshift-config-managed] adding namespace took 7.171223ms I0327 15:09:18.996599 3959 namespace.go:98] [openshift] adding namespace I0327 15:09:18.997295 3959 namespace.go:102] [openshift-network-console] adding namespace took 7.328447ms I0327 15:09:18.997310 3959 namespace.go:98] [default] adding namespace I0327 15:09:18.997314 3959 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:18.997863 3959 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.54265ms I0327 15:09:18.997879 3959 namespace.go:98] [openshift-apiserver] adding namespace I0327 15:09:18.998385 3959 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.682707ms I0327 15:09:18.998400 3959 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 15:09:18.998924 3959 namespace.go:102] [openshift-infra] adding namespace took 7.79912ms I0327 15:09:18.998939 3959 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 15:09:18.999329 3959 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.840309ms I0327 15:09:18.999346 3959 namespace.go:98] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] adding namespace I0327 15:09:18.999877 3959 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.999493ms I0327 15:09:18.999891 3959 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 15:09:19.000301 3959 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.597171ms I0327 15:09:19.000766 3959 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.608ms I0327 15:09:19.001344 3959 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.338277ms I0327 15:09:19.001364 3959 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 15:09:19.001891 3959 namespace.go:102] [openshift-image-registry] adding namespace took 7.263365ms I0327 15:09:19.002459 3959 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.264426ms I0327 15:09:19.002959 3959 namespace.go:102] [openshift-backplane] adding namespace took 7.367019ms I0327 15:09:19.002976 3959 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 15:09:19.003525 3959 namespace.go:102] [openshift-console-operator] adding namespace took 7.37281ms I0327 15:09:19.003983 3959 namespace.go:102] [openshift] adding namespace took 7.377631ms I0327 15:09:19.004359 3959 namespace.go:102] [default] adding namespace took 7.042685ms I0327 15:09:19.004789 3959 namespace.go:102] [openshift-apiserver] adding namespace took 6.904191ms I0327 15:09:19.005196 3959 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.789901ms I0327 15:09:19.005209 3959 namespace.go:98] [openshift-monitoring] adding namespace I0327 15:09:19.005579 3959 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.634463ms I0327 15:09:19.006033 3959 namespace.go:102] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] adding namespace took 6.682393ms I0327 15:09:19.006427 3959 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.530385ms I0327 15:09:19.006858 3959 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 5.487083ms I0327 15:09:19.007323 3959 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 4.342327ms I0327 15:09:19.007691 3959 namespace.go:102] [openshift-monitoring] adding namespace took 2.477132ms I0327 15:09:19.008118 3959 zone_ic_handler.go:186] Time taken to create transit switch: 312.961µs I0327 15:09:19.008221 3959 master.go:532] Adding or Updating local node "ip-10-0-1-144.ec2.internal" for network "default" I0327 15:09:19.008257 3959 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-192.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-192.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-192.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-192.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-192.ec2.internal I0327 15:09:19.008269 3959 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-34.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-34.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-34.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-34.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-34.ec2.internal W0327 15:09:19.016505 3959 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-4cpwh: pod kube-system/global-pull-secret-syncer-4cpwh: no pod IPs found W0327 15:09:19.016529 3959 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-xrnlc: pod openshift-network-diagnostics/network-check-target-xrnlc: no pod IPs found W0327 15:09:19.016542 3959 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-64l54: pod openshift-multus/network-metrics-daemon-64l54: no pod IPs found I0327 15:09:19.017535 3959 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-144.ec2.internal for the network default I0327 15:09:19.018202 3959 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 15:09:19.018234 3959 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 15:09:19.018249 3959 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 15:09:19.018270 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:19.018311 3959 node_tracker.go:235] Node ip-10-0-2-34.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-34.ec2.internal" I0327 15:09:19.018329 3959 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:19.018342 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:19.018349 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-192.ec2.internal in network "default" I0327 15:09:19.018365 3959 node_tracker.go:235] Node ip-10-0-0-192.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-192.ec2.internal" I0327 15:09:19.018375 3959 node_tracker.go:169] Node ip-10-0-0-192.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:19.018381 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:19.018393 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-144.ec2.internal in network "default" I0327 15:09:19.018431 3959 node_tracker.go:169] Node ip-10-0-1-144.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:19.018440 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:19.118866 3959 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 15:09:19.118903 3959 services_controller.go:208] Setting up event handlers for services for network=default I0327 15:09:19.118962 3959 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 15:09:19.119006 3959 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 15:09:19.119030 3959 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 15:09:19.219266 3959 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 15:09:19.219289 3959 repair.go:60] Starting repairing loop for services I0327 15:09:19.219379 3959 repair.go:131] Deleted 0 stale service LBs I0327 15:09:19.219400 3959 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 15:09:19.219416 3959 repair.go:62] Finished repairing loop for services: 128.532µs I0327 15:09:19.219456 3959 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 15:09:19.219466 3959 services_controller.go:254] Starting workers for network=default I0327 15:09:19.219856 3959 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-4cpwh] creating logical port kube-system_global-pull-secret-syncer-4cpwh for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:19.219900 3959 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-nfpdk, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nfpdk: suppressed error logged: pod openshift-multus/network-metrics-daemon-nfpdk: no pod IPs found I0327 15:09:19.219904 3959 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-dr7nn, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-dr7nn: suppressed error logged: pod openshift-multus/network-metrics-daemon-dr7nn: no pod IPs found I0327 15:09:19.219940 3959 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-8qmwh, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-8qmwh: suppressed error logged: pod kube-system/global-pull-secret-syncer-8qmwh: no pod IPs found I0327 15:09:19.219966 3959 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-64l54] creating logical port openshift-multus_network-metrics-daemon-64l54 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:19.220036 3959 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xrnlc] creating logical port openshift-network-diagnostics_network-check-target-xrnlc for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:19.220069 3959 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-4cpwh I0327 15:09:19.220077 3959 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rwdfv: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rwdfv: no pod IPs found I0327 15:09:19.220103 3959 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-64l54 I0327 15:09:19.220059 3959 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-dcfk8: suppressed error logged: pod openshift-network-diagnostics/network-check-target-dcfk8: no pod IPs found I0327 15:09:19.220181 3959 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-xrnlc I0327 15:09:19.237306 3959 pod.go:62] [openshift-network-diagnostics/network-check-target-xrnlc] pod update took 17.150405ms I0327 15:09:19.237324 3959 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-xrnlc] addLogicalPort annotation time took 17.187099ms I0327 15:09:19.237345 3959 pod.go:62] [kube-system/global-pull-secret-syncer-4cpwh] pod update took 17.406427ms I0327 15:09:19.237358 3959 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-4cpwh] addLogicalPort annotation time took 17.424369ms I0327 15:09:19.237924 3959 pod.go:62] [openshift-multus/network-metrics-daemon-64l54] pod update took 17.897375ms I0327 15:09:19.237940 3959 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-64l54] addLogicalPort annotation time took 17.914737ms I0327 15:09:19.238160 3959 pods.go:271] [openshift-network-diagnostics/network-check-target-xrnlc] addLogicalPort took 18.130387ms, libovsdb time 639.32µs I0327 15:09:19.238558 3959 pods.go:271] [kube-system/global-pull-secret-syncer-4cpwh] addLogicalPort took 18.714664ms, libovsdb time 1.019667ms I0327 15:09:19.238831 3959 pods.go:271] [openshift-multus/network-metrics-daemon-64l54] addLogicalPort took 18.892879ms, libovsdb time 599.76µs I0327 15:09:19.239178 3959 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 15:09:19.239195 3959 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 15:09:19.239205 3959 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 15:09:19.239209 3959 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 15:09:19.239394 3959 repair.go:29] Repairing admin network policies took 179.784µs I0327 15:09:19.239515 3959 repair.go:92] Repairing baseline admin network policies took 109.688µs I0327 15:09:19.239907 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 15:09:19.240288 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 15:09:19.241826 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:09:19.241844 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 15:09:19.241854 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 15:09:19.243311 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 15:09:19.243329 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 15:09:19.243336 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 15:09:19.243591 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 15:09:19.243607 3959 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 15:09:19.243617 3959 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 15:09:19.245783 3959 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 15:09:19.245800 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 15:09:19.245808 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 15:09:19.246291 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 15:09:19.246313 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 15:09:19.246757 3959 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 15:09:19.246771 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:09:19.247435 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 15:09:19.247452 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 15:09:19.247459 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 15:09:19.248637 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 15:09:19.248672 3959 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 15:09:19.248683 3959 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 15:09:19.250630 3959 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 15:09:19.250648 3959 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 15:09:19.250673 3959 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 15:09:19.251445 3959 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 15:09:19.251463 3959 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 15:09:19.251475 3959 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 15:09:19.253006 3959 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 15:09:19.253027 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 15:09:19.253035 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 15:09:19.253516 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 15:09:19.253534 3959 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 15:09:19.253542 3959 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 15:09:19.255595 3959 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 15:09:19.255613 3959 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 15:09:19.255620 3959 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 15:09:19.255960 3959 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 15:09:19.255978 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 15:09:19.255985 3959 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 15:09:19.257386 3959 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 15:09:19.257403 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 15:09:19.257411 3959 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 15:09:19.257680 3959 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 15:09:19.257698 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 15:09:19.257729 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:09:19.258396 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 15:09:19.258410 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 15:09:19.258419 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 15:09:19.258960 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:09:19.258979 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 15:09:19.258986 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 15:09:19.259436 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 15:09:19.259452 3959 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 15:09:19.259459 3959 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 15:09:19.259953 3959 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 15:09:19.259968 3959 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 15:09:19.259974 3959 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 15:09:19.260230 3959 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 15:09:19.260248 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 15:09:19.260256 3959 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 15:09:19.260746 3959 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 15:09:19.260767 3959 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 15:09:19.260777 3959 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 15:09:19.261788 3959 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 15:09:19.261805 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 15:09:19.261812 3959 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 15:09:19.262268 3959 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 15:09:19.262287 3959 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 15:09:19.262298 3959 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 15:09:19.262767 3959 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 15:09:19.262781 3959 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 15:09:19.262790 3959 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 15:09:19.263613 3959 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 15:09:19.263631 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 15:09:19.263640 3959 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 15:09:19.264312 3959 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 15:09:19.264328 3959 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 15:09:19.264336 3959 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 15:09:19.264829 3959 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 15:09:19.264846 3959 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 15:09:19.264855 3959 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 15:09:19.265862 3959 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 15:09:19.265881 3959 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 15:09:19.265891 3959 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 15:09:19.267910 3959 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 15:09:19.272911 3959 egressfirewall.go:319] Starting EgressFirewall controller I0327 15:09:19.272943 3959 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 15:09:19.272970 3959 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 15:09:19.272982 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 15:09:19.273002 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 15:09:19.273012 3959 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 15:09:19.273003 3959 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 15:09:19.273290 3959 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 15:09:19.273305 3959 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 15:09:19.273314 3959 egressqos.go:193] Setting up event handlers for EgressQoS I0327 15:09:19.273398 3959 egressfirewall.go:628] Syncing node "ip-10-0-0-192.ec2.internal" for egress firewall I0327 15:09:19.273411 3959 egressfirewall.go:628] Syncing node "ip-10-0-1-144.ec2.internal" for egress firewall I0327 15:09:19.273415 3959 egressfirewall.go:628] Syncing node "ip-10-0-2-34.ec2.internal" for egress firewall I0327 15:09:19.273448 3959 egressqos.go:245] Starting EgressQoS Controller I0327 15:09:19.273461 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 15:09:19.273469 3959 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 15:09:19.273474 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 15:09:19.273479 3959 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 15:09:19.273483 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 15:09:19.273487 3959 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 15:09:19.273491 3959 egressqos.go:259] Repairing EgressQoSes I0327 15:09:19.273495 3959 egressqos.go:399] Starting repairing loop for egressqos I0327 15:09:19.273572 3959 egressqos.go:401] Finished repairing loop for egressqos: 75.379µs I0327 15:09:19.273587 3959 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 15:09:19.273678 3959 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-144.ec2.internal I0327 15:09:19.273743 3959 egressservice_zone.go:210] Starting Egress Services Controller I0327 15:09:19.273749 3959 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-144.ec2.internal : 67.015µs I0327 15:09:19.273761 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:09:19.273773 3959 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:09:19.273785 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:09:19.273793 3959 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:09:19.273804 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:09:19.273813 3959 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:09:19.273824 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 15:09:19.273832 3959 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 15:09:19.273839 3959 egressservice_zone.go:228] Repairing Egress Services I0327 15:09:19.274122 3959 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 15:09:19.274131 3959 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:09:19.274192 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-144.ec2.internal I0327 15:09:19.274205 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-144.ec2.internal: 13.028µs I0327 15:09:19.274216 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:09:19.274223 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 7.255µs I0327 15:09:19.274230 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:19.274236 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 6.523µs I0327 15:09:19.274241 3959 default_network_controller.go:568] Completing all the Watchers took 303.928449ms I0327 15:09:19.274252 3959 default_network_controller.go:572] Starting unidling controllers I0327 15:09:19.274263 3959 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 15:09:19.274268 3959 unidle.go:63] Populating Initial ContollerEvent events I0327 15:09:19.274278 3959 unidle.go:79] Setting up event handlers for services I0327 15:09:19.274349 3959 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:09:19.498378 3959 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:19.998022 3959 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:20.498196 3959 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:20.998695 3959 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:21.025352 3959 egressfirewall.go:628] Syncing node "ip-10-0-0-192.ec2.internal" for egress firewall I0327 15:09:21.025380 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:21.025396 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 21.535µs I0327 15:09:21.025420 3959 default_network_controller.go:1008] Node "ip-10-0-0-192.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:21.445577 3959 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:09:21.499291 3959 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:21.515997 3959 bridgeconfig.go:537] Gateway is ready I0327 15:09:21.516021 3959 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 15:09:21.526890 3959 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.144. Updating OVN encap IP. I0327 15:09:21.531794 3959 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.144 - it is already configured I0327 15:09:21.531808 3959 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.144/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 15:09:21.532054 3959 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.144/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-144.ec2.internal","mac-address":"0a:ff:d3:61:56:43","ip-addresses":["10.0.1.144/24"],"ip-address":"10.0.1.144/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:15418104-dcbb-4f24-b50d-233cff74ddc6 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.144/24"}] on node ip-10-0-1-144.ec2.internal I0327 15:09:21.547232 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-144.ec2.internal I0327 15:09:21.547249 3959 master.go:532] Adding or Updating local node "ip-10-0-1-144.ec2.internal" for network "default" I0327 15:09:21.547347 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-144.ec2.internal in network "default" I0327 15:09:21.547407 3959 node_tracker.go:169] Node ip-10-0-1-144.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:21.547445 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:21.547251 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-144.ec2.internal: 25.185µs I0327 15:09:21.547349 3959 egressfirewall.go:628] Syncing node "ip-10-0-1-144.ec2.internal" for egress firewall I0327 15:09:21.547617 3959 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.144 I0327 15:09:21.547745 3959 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-144.ec2.internal I0327 15:09:21.552123 3959 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-144.ec2.internal. Updating no re-route policies I0327 15:09:21.552215 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-wvvlh I0327 15:09:21.552223 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-42zpd I0327 15:09:21.552234 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-wvvlh I0327 15:09:21.552238 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-86nxt I0327 15:09:21.552251 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-86nxt I0327 15:09:21.552255 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-wvvlh after 0 failed attempt(s) I0327 15:09:21.552254 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8 I0327 15:09:21.552265 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-dr7nn I0327 15:09:21.552269 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8 I0327 15:09:21.552268 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5v9l4 I0327 15:09:21.552275 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-dr7nn I0327 15:09:21.552272 3959 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-144.ec2.internal I0327 15:09:21.552277 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv I0327 15:09:21.552286 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-64l54 I0327 15:09:21.552290 3959 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-144.ec2.internal I0327 15:09:21.552292 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv I0327 15:09:21.552295 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-64l54 I0327 15:09:21.552298 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-9gb24 I0327 15:09:21.552297 3959 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-8qmwh I0327 15:09:21.552304 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-9cbjq I0327 15:09:21.552307 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-9gb24 I0327 15:09:21.552308 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xrnlc I0327 15:09:21.552309 3959 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-dr7nn, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-dr7nn: suppressed error logged: pod openshift-multus/network-metrics-daemon-dr7nn: no pod IPs found I0327 15:09:21.552314 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-9cbjq I0327 15:09:21.552316 3959 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-8qmwh I0327 15:09:21.552320 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xrnlc I0327 15:09:21.552328 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-9gb24 after 0 failed attempt(s) I0327 15:09:21.552329 3959 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rwdfv: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rwdfv: no pod IPs found I0327 15:09:21.552267 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-144.ec2.internal I0327 15:09:21.552333 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-9cbjq after 0 failed attempt(s) I0327 15:09:21.552338 3959 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-64l54] creating logical port openshift-multus_network-metrics-daemon-64l54 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:21.552342 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-144.ec2.internal I0327 15:09:21.552353 3959 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xrnlc] creating logical port openshift-network-diagnostics_network-check-target-xrnlc for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:21.552354 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-144.ec2.internal after 0 failed attempt(s) I0327 15:09:21.552356 3959 obj_retry.go:375] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-8qmwh, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-8qmwh: suppressed error logged: pod kube-system/global-pull-secret-syncer-8qmwh: no pod IPs found I0327 15:09:21.552293 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nfpdk I0327 15:09:21.552389 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nfpdk I0327 15:09:21.552432 3959 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-nfpdk, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nfpdk: suppressed error logged: pod openshift-multus/network-metrics-daemon-nfpdk: no pod IPs found I0327 15:09:21.552240 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-42zpd I0327 15:09:21.552275 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-86nxt after 0 failed attempt(s) I0327 15:09:21.552252 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-vnpnx I0327 15:09:21.552474 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-vnpnx I0327 15:09:21.552494 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-vnpnx after 0 failed attempt(s) I0327 15:09:21.552313 3959 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-144.ec2.internal after 0 failed attempt(s) I0327 15:09:21.552301 3959 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-dcfk8: suppressed error logged: pod openshift-network-diagnostics/network-check-target-dcfk8: no pod IPs found I0327 15:09:21.552239 3959 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-4wj4p I0327 15:09:21.552522 3959 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-4wj4p I0327 15:09:21.552539 3959 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-4wj4p after 0 failed attempt(s) I0327 15:09:21.552253 3959 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-4cpwh I0327 15:09:21.552553 3959 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-4cpwh I0327 15:09:21.552579 3959 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-4cpwh] creating logical port kube-system_global-pull-secret-syncer-4cpwh for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:21.552278 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5v9l4 I0327 15:09:21.552647 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5v9l4 after 0 failed attempt(s) I0327 15:09:21.552226 3959 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-gvvvc I0327 15:09:21.552678 3959 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-gvvvc I0327 15:09:21.552690 3959 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-gvvvc after 0 failed attempt(s) I0327 15:09:21.552311 3959 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-m8b8k I0327 15:09:21.552700 3959 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-m8b8k I0327 15:09:21.552712 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-m8b8k after 0 failed attempt(s) I0327 15:09:21.552451 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-42zpd after 0 failed attempt(s) I0327 15:09:21.553086 3959 pods.go:271] [openshift-multus/network-metrics-daemon-64l54] addLogicalPort took 757.386µs, libovsdb time 294.239µs I0327 15:09:21.553095 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-64l54 after 0 failed attempt(s) I0327 15:09:21.553224 3959 pods.go:271] [openshift-network-diagnostics/network-check-target-xrnlc] addLogicalPort took 878.338µs, libovsdb time 349.321µs I0327 15:09:21.553241 3959 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-xrnlc after 0 failed attempt(s) I0327 15:09:21.553639 3959 pods.go:271] [kube-system/global-pull-secret-syncer-4cpwh] addLogicalPort took 1.067861ms, libovsdb time 672.363µs I0327 15:09:21.553652 3959 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-4cpwh after 0 failed attempt(s) I0327 15:09:21.565037 3959 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 15:09:21.576160 3959 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 15:09:21.579417 3959 iptables.go:108] Creating table: nat chain: OUTPUT I0327 15:09:21.601736 3959 iptables.go:108] Creating table: nat chain: PREROUTING I0327 15:09:21.677528 3959 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:09:21.695076 3959 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 15:09:21.695209 3959 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 15:09:21.695263 3959 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 15:09:21.695362 3959 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 15:09:21.695413 3959 gateway_init.go:325] Gateway and management port readiness took 2.701233225s I0327 15:09:21.695442 3959 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 15:09:21.695474 3959 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 15:09:21.695484 3959 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 15:09:21.695493 3959 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:09:21.695548 3959 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 15:09:21.695645 3959 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 60.327µs I0327 15:09:21.695831 3959 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 15:09:21.695862 3959 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 21.534µs I0327 15:09:21.695870 3959 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 40.944µs I0327 15:09:21.695876 3959 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 15:09:21.695886 3959 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 15:09:21.695891 3959 default_node_network_controller.go:898] Starting the default node network controller I0327 15:09:21.695896 3959 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 15:09:21.695978 3959 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 17.543µs I0327 15:09:21.696022 3959 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 15:09:21.696030 3959 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 15:09:21.696060 3959 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b81260 0xc0002ddc70 0xc0002ddce0 0xc0002ddd50 0xc0002dddc0 0xc0002dde30 0xc0002ddea0 0xc000b975e0 0xc0002ddf10 0xc0002ec000 0xc0009ae3c0 0xc0064f9180 0xc0002ec070 map[0x2f67040:0xc002d13680 0x2f67580:0xc002d13800 0x2f68000:0xc002d135c0 0x2f682a0:0xc002d13620 0x2f687e0:0xc002d136e0 0x2f68a80:0xc002d13740 0x2f68d20:0xc002d137a0 0x2f69500:0xc002d13c20 0x2f697a0:0xc002d13c80 0x2f93d40:0xc001214ba0 0x2f940e0:0xc001214c00 0x2f94bc0:0xc000bf5860 0x2f96520:0xc001bce000 0x2f97000:0xc001214de0 0x2f97e80:0xc001bce060] 0xc000ba8690 0} I0327 15:09:21.696114 3959 gateway.go:151] Starting gateway service sync I0327 15:09:21.696532 3959 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 15:09:21.698654 3959 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 15:09:21.718043 3959 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 15:09:21.719222 3959 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 15:09:21.720413 3959 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 15:09:21.754690 3959 gateway.go:168] Gateway service sync done. Time taken: 58.561137ms I0327 15:09:21.755173 3959 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31433 I0327 15:09:21.755514 3959 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b81260 0xc0002ddc70 0xc0002ddce0 0xc0002ddd50 0xc0002dddc0 0xc0002dde30 0xc0002ddea0 0xc000b975e0 0xc0002ddf10 0xc0002ec000 0xc0009ae3c0 0xc0064f9180 0xc0002ec070 map[0x2f67040:0xc002d13680 0x2f67580:0xc002d13800 0x2f68000:0xc002d135c0 0x2f682a0:0xc002d13620 0x2f687e0:0xc002d136e0 0x2f68a80:0xc002d13740 0x2f68d20:0xc002d137a0 0x2f69500:0xc002d13c20 0x2f697a0:0xc002d13c80 0x2f93d40:0xc001214ba0 0x2f940e0:0xc001214c00 0x2f94bc0:0xc000bf5860 0x2f96520:0xc001bce000 0x2f97000:0xc001214de0 0x2f97e80:0xc001bce060] 0xc000ba8690 0} I0327 15:09:21.755996 3959 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b81260 0xc0002ddc70 0xc0002ddce0 0xc0002ddd50 0xc0002dddc0 0xc0002dde30 0xc0002ddea0 0xc000b975e0 0xc0002ddf10 0xc0002ec000 0xc0009ae3c0 0xc0064f9180 0xc0002ec070 map[0x2f67040:0xc002d13680 0x2f67580:0xc002d13800 0x2f68000:0xc002d135c0 0x2f682a0:0xc002d13620 0x2f687e0:0xc002d136e0 0x2f68a80:0xc002d13740 0x2f68d20:0xc002d137a0 0x2f69500:0xc002d13c20 0x2f697a0:0xc002d13c80 0x2f93d40:0xc001214ba0 0x2f940e0:0xc001214c00 0x2f94bc0:0xc000bf5860 0x2f96520:0xc001bce000 0x2f97000:0xc001214de0 0x2f97e80:0xc001bce060] 0xc000ba8690 0} I0327 15:09:21.756037 3959 gateway.go:519] Reconciling gateway with updates I0327 15:09:21.761713 3959 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 15:09:21.761735 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:21.761733 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:21.761753 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:21.761758 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:21.761764 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:21.761768 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:21.761770 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:21.761777 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:21.761782 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:21.761790 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:21.761768 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:21.761798 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:21.761802 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:09:21.761798 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:21.761814 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:21.761814 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:21.761815 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:21.761826 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:21.761830 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:21.761836 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:09:21.761839 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:21.761836 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:21.761844 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761853 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:21.761781 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:21.761860 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:09:21.761857 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:21.761775 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:21.761873 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:21.761876 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:09:21.761881 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:21.761885 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:21.761888 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:21.761893 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:21.761896 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:21.761897 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:21.761902 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:21.761901 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:21.761792 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:21.761890 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:21.761931 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:09:21.761940 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:09:21.761943 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761861 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:21.761950 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:09:21.761954 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:21.761960 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:21.761960 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:21.761965 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:09:21.761967 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:21.761792 3959 port_claim.go:71] Svc openshift-ingress/router-default: port 32501 is already open, no action needed I0327 15:09:21.761971 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:21.761887 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:21.761978 3959 default_node_network_controller.go:943] Gateway and management port readiness took 65.938128ms I0327 15:09:21.761986 3959 port_claim.go:71] Svc openshift-ingress/router-default: port 32216 is already open, no action needed I0327 15:09:21.761991 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:21.762003 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761874 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:21.761960 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:21.762044 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761896 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:21.761846 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:21.762059 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:21.762061 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:21.762043 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:09:21.761844 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:21.761875 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:21.761927 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:21.761874 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:09:21.762228 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:09:21.761901 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:21.761914 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:09:21.761821 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:09:21.761782 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:21.761915 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:21.762311 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:21.761917 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:21.762329 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:09:21.761930 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:21.762358 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:21.761760 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:21.761918 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:21.762416 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:21.762434 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:21.762422 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:21.762442 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:09:21.762440 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:21.762460 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:21.762464 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:21.762477 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:21.762483 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:21.762462 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:21.762541 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761949 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:21.762554 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:21.762559 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761952 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:21.762569 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:21.761961 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:21.761964 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:21.761962 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:21.762634 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:21.762639 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:09:21.761944 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:09:21.762647 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:09:21.762655 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:09:21.761970 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:21.762051 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:21.761943 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:21.762779 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:21.761937 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:09:21.761947 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:09:21.763321 3959 node_ip_handler_linux.go:160] Node IP manager is running W0327 15:09:21.800317 3959 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:09:21.800459 3959 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:09:21.800555 3959 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0327 15:09:21.808248 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:09:21.827856 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:09:21.827866 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:09:21.827884 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:09:21.827893 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:09:21.827900 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:09:21.827911 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:09:21.827922 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:09:21.827939 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:09:21.827957 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:09:21.827964 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:09:21.866867 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:09:21.866900 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:09:21.866908 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:09:21.866916 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:09:21.866925 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:09:21.866935 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:09:21.866952 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:09:21.866970 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:09:21.866976 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:09:21.866983 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:09:21.866988 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:09:21.866994 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:09:22.911398 3959 default_node_network_controller.go:1364] Starting node controller node sync W0327 15:09:22.911435 3959 egressip_healthcheck.go:76] Health checking using insecure connection I0327 15:09:22.911481 3959 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0327 15:09:22.923008 3959 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.585299ms I0327 15:09:22.923228 3959 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 15:09:22.923509 3959 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 15:09:22.923619 3959 egressservice_node.go:183] Starting Egress Services Controller I0327 15:09:22.923634 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:09:22.923642 3959 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:09:22.923647 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:09:22.923652 3959 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:09:22.923670 3959 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:09:22.923681 3959 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:09:22.923689 3959 egressservice_node.go:197] Repairing Egress Services I0327 15:09:22.933492 3959 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 15:09:22.933504 3959 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:09:22.933651 3959 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 15:09:22.933722 3959 link_network_manager.go:117] Link manager is running I0327 15:09:22.933737 3959 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 15:09:22.933828 3959 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 15:09:22.934234 3959 vrf_manager.go:128] VRF manager is running I0327 15:09:22.934339 3959 node_controller_manager.go:437] Removing flows to drop GARP I0327 15:09:22.934353 3959 gateway.go:519] Reconciling gateway with updates I0327 15:09:22.964579 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:22.964601 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:09:22.964603 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:22.964611 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:22.964613 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:22.964617 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:22.964617 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:22.964623 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:22.964622 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:22.964631 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:22.964637 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:22.964640 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:22.964642 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:09:22.964650 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:09:22.964588 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:22.964675 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:22.964683 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:22.964684 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:22.964691 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:22.964696 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:22.964696 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:22.964696 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:22.964703 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:09:22.964706 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:22.964707 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:22.964710 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:22.964713 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:22.964719 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:09:22.964717 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:22.964614 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:09:22.964701 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:09:22.964728 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:22.964734 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:22.964708 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:09:22.964741 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:22.964741 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:09:22.964604 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:22.964749 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:22.964738 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:22.964761 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:22.964762 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:09:22.964770 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:22.964719 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:22.964778 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:22.964781 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:22.964632 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:22.964791 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:22.964793 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:22.964796 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:22.964800 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:22.964623 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:22.964807 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:22.964806 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:22.964808 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:22.964812 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:09:22.964801 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:22.964817 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:22.964719 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:22.964827 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:22.964757 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:22.964766 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:09:22.964725 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:09:22.964736 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:22.964725 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:09:22.964760 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:22.964858 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:09:22.964736 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:22.964866 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:09:22.964646 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:22.964882 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:22.964605 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:22.964923 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:22.964928 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:09:22.964700 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:22.964936 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:22.964710 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:22.964756 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:22.965031 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:22.964774 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:22.965085 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:22.965095 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:09:22.964784 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:22.965108 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:22.964785 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:22.965146 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:22.964786 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:22.965201 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:22.964793 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:22.965211 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:22.965214 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:22.965218 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:22.965223 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:22.965231 3959 port_claim.go:71] Svc openshift-ingress/router-default: port 32501 is already open, no action needed I0327 15:09:22.964796 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:09:22.965239 3959 port_claim.go:71] Svc openshift-ingress/router-default: port 32216 is already open, no action needed I0327 15:09:22.965241 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:09:22.965245 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:09:22.964687 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:22.965259 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:22.964819 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:22.964744 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:22.964871 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:22.964590 3959 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:22.965351 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:22.965358 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:09:22.965401 3959 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:23.006109 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:09:23.006145 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006169 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:09:23.006189 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:09:23.006202 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006217 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:09:23.006227 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:09:23.006239 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006249 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006260 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:09:23.006270 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:09:23.006282 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006291 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:09:23.006301 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:09:23.006310 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006322 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:09:23.006331 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:09:23.006345 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:09:23.006364 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:09:23.006372 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:09:23.006379 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:09:23.047045 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:09:23.065923 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:09:23.065932 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:09:23.065947 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:09:23.065955 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:09:23.065961 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:09:23.065966 3959 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:09:24.839676 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-144.ec2.internal I0327 15:09:24.839711 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-144.ec2.internal: 55.327µs I0327 15:09:24.869163 3959 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c85498c95-q75rf] creating logical port openshift-image-registry_image-registry-7c85498c95-q75rf for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:24.869318 3959 kube.go:256] Updating pod openshift-image-registry/image-registry-7c85498c95-q75rf I0327 15:09:24.890607 3959 pod.go:62] [openshift-image-registry/image-registry-7c85498c95-q75rf] pod update took 21.281547ms I0327 15:09:24.890627 3959 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7c85498c95-q75rf] addLogicalPort annotation time took 21.355581ms I0327 15:09:24.892079 3959 pods.go:271] [openshift-image-registry/image-registry-7c85498c95-q75rf] addLogicalPort took 22.933807ms, libovsdb time 1.110201ms I0327 15:09:24.905556 3959 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-mrt9w] creating logical port openshift-dns_dns-default-mrt9w for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:24.905570 3959 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-xpx9l] creating logical port openshift-ingress-canary_ingress-canary-xpx9l for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:24.905698 3959 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-xpx9l I0327 15:09:24.905757 3959 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-xz9jb for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:24.905699 3959 kube.go:256] Updating pod openshift-dns/dns-default-mrt9w I0327 15:09:24.905860 3959 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb I0327 15:09:24.922307 3959 pod.go:62] [openshift-dns/dns-default-mrt9w] pod update took 16.653054ms I0327 15:09:24.922336 3959 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-mrt9w] addLogicalPort annotation time took 16.687586ms I0327 15:09:24.922383 3959 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb] pod update took 16.55727ms I0327 15:09:24.922428 3959 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb] addLogicalPort annotation time took 16.577976ms I0327 15:09:24.922458 3959 pod.go:62] [openshift-ingress-canary/ingress-canary-xpx9l] pod update took 16.805929ms I0327 15:09:24.922478 3959 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-xpx9l] addLogicalPort annotation time took 16.829819ms I0327 15:09:24.923850 3959 pods.go:271] [openshift-dns/dns-default-mrt9w] addLogicalPort took 18.313251ms, libovsdb time 1.121595ms I0327 15:09:24.924293 3959 pods.go:271] [openshift-ingress-canary/ingress-canary-xpx9l] addLogicalPort took 18.737752ms, libovsdb time 1.556783ms I0327 15:09:24.924771 3959 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb] addLogicalPort took 19.020043ms, libovsdb time 1.989819ms I0327 15:09:25.289870 3959 cni.go:403] [openshift-ingress-canary/ingress-canary-xpx9l 3c6ce4c42c61ce16c295bb14200295cb457ca15b4a6b199735a866e71ba85863 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-xpx9l 3c6ce4c42c61ce16c295bb14200295cb457ca15b4a6b199735a866e71ba85863 network default NAD default] I0327 15:09:25.306848 3959 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb 3f31ce36d39b695a272b907985aad3a647322e3f8e0f7bedf925d9cf04d6a1eb network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb 3f31ce36d39b695a272b907985aad3a647322e3f8e0f7bedf925d9cf04d6a1eb network default NAD default] I0327 15:09:25.329408 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-xpx9l, hostIfaceName: 3c6ce4c42c61ce1, network: default, NAD default, SandboxID: "3c6ce4c42c61ce16c295bb14200295cb457ca15b4a6b199735a866e71ba85863", PCI device ID: , UID: "0c56860b-1796-4743-b799-85b4d65b6666", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0327 15:09:25.329769 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-xz9jb, hostIfaceName: 3f31ce36d39b695, network: default, NAD default, SandboxID: "3f31ce36d39b695a272b907985aad3a647322e3f8e0f7bedf925d9cf04d6a1eb", PCI device ID: , UID: "f9384a24-c38f-4b93-ad0c-9e1649125cce", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0327 15:09:25.378102 3959 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb 3f31ce36d39b695a272b907985aad3a647322e3f8e0f7bedf925d9cf04d6a1eb network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jb 3f31ce36d39b695a272b907985aad3a647322e3f8e0f7bedf925d9cf04d6a1eb network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:7e:46:e5:d1:07\",\"name\":\"3f31ce36d39b695\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/187079b9-d515-423c-9429-c0bde206de11\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:09:25.379087 3959 cni.go:424] [openshift-ingress-canary/ingress-canary-xpx9l 3c6ce4c42c61ce16c295bb14200295cb457ca15b4a6b199735a866e71ba85863 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-xpx9l 3c6ce4c42c61ce16c295bb14200295cb457ca15b4a6b199735a866e71ba85863 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:98:03:5a:bb:9f\",\"name\":\"3c6ce4c42c61ce1\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50a6ec4f-f1ef-42be-a41d-69e3eaf2c43a\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:09:25.521419 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-192.ec2.internal in network "default" I0327 15:09:25.521467 3959 node_tracker.go:235] Node ip-10-0-0-192.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-192.ec2.internal" I0327 15:09:25.521485 3959 node_tracker.go:169] Node ip-10-0-0-192.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:25.521503 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:25.521565 3959 default_network_controller.go:1008] Node "ip-10-0-0-192.ec2.internal" in remote zone "ip-10-0-0-192.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:25.601292 3959 default_network_controller.go:1008] Node "ip-10-0-0-192.ec2.internal" in remote zone "ip-10-0-0-192.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:25.629453 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-192.ec2.internal in network "default" I0327 15:09:25.629502 3959 default_network_controller.go:1008] Node "ip-10-0-0-192.ec2.internal" in remote zone "ip-10-0-0-192.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:25.629515 3959 node_tracker.go:169] Node ip-10-0-0-192.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:25.629541 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:25.630227 3959 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-192.ec2.internal for the network default I0327 15:09:25.631640 3959 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-192.ec2.internal" on network "default" took: 1.424333ms I0327 15:09:28.181386 3959 egressfirewall.go:628] Syncing node "ip-10-0-0-192.ec2.internal" for egress firewall I0327 15:09:28.181442 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-192.ec2.internal in network "default" I0327 15:09:28.181463 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:28.181481 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 23.462µs I0327 15:09:28.181501 3959 node_tracker.go:169] Node ip-10-0-0-192.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:28.181519 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:28.181522 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-192.ec2.internal", IP: 10.0.0.192 to PMTUD blocking rules I0327 15:09:28.181531 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-192.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:09:28.188042 3959 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-192.ec2.internal. Updating no re-route policies I0327 15:09:28.196459 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-192.ec2.internal", IP: 10.0.0.192 to PMTUD blocking rules I0327 15:09:28.196484 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-192.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:09:31.816775 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:31.817047 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 279.044µs W0327 15:09:31.851858 3959 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:32.460551 3959 cni.go:403] [kube-system/global-pull-secret-syncer-4cpwh cb0d308b16ba153832fee56d31122aec62cb28a864f2e518c81f1c51151997ae network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-4cpwh cb0d308b16ba153832fee56d31122aec62cb28a864f2e518c81f1c51151997ae network default NAD default] I0327 15:09:32.469560 3959 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-4cpwh, hostIfaceName: cb0d308b16ba153, network: default, NAD default, SandboxID: "cb0d308b16ba153832fee56d31122aec62cb28a864f2e518c81f1c51151997ae", PCI device ID: , UID: "c6841814-13b4-4435-b8df-2de976325d89", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0327 15:09:32.514275 3959 cni.go:424] [kube-system/global-pull-secret-syncer-4cpwh cb0d308b16ba153832fee56d31122aec62cb28a864f2e518c81f1c51151997ae network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-4cpwh cb0d308b16ba153832fee56d31122aec62cb28a864f2e518c81f1c51151997ae network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:f5:e7:45:fd:2a\",\"name\":\"cb0d308b16ba153\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/629adad3-9d03-47f5-9fe4-7d2ce3bcd116\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:09:36.879360 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:37.890932 3959 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:37.890975 3959 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.373601 3959 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:38.883098 3959 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.883127 3959 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:09:40.179438 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:42.096320 3959 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "ip-10-0-2-34.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:42.096340 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:42.096367 3959 node_tracker.go:235] Node ip-10-0-2-34.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-34.ec2.internal" I0327 15:09:42.096383 3959 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:42.096399 3959 services_controller.go:574] Full service sync requested for network=default W0327 15:09:42.096759 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:42.145216 3959 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "ip-10-0-2-34.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:42.225680 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:42.225740 3959 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:42.225755 3959 services_controller.go:574] Full service sync requested for network=default I0327 15:09:42.225683 3959 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "ip-10-0-2-34.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0327 15:09:42.225962 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:42.226547 3959 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-34.ec2.internal for the network default I0327 15:09:42.227605 3959 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-34.ec2.internal" on network "default" took: 1.070205ms I0327 15:09:44.779020 3959 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:44.779030 3959 egressfirewall.go:628] Syncing node "ip-10-0-2-34.ec2.internal" for egress firewall I0327 15:09:44.779044 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:09:44.779057 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 17.887µs I0327 15:09:44.779058 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:09:44.779070 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 10.0.2.34 to PMTUD blocking rules I0327 15:09:44.779085 3959 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:44.779101 3959 services_controller.go:574] Full service sync requested for network=default W0327 15:09:44.779372 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:44.785426 3959 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-34.ec2.internal. Updating no re-route policies I0327 15:09:44.795259 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 10.0.2.34 to PMTUD blocking rules I0327 15:09:44.795274 3959 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:09:47.697838 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:09:47.697872 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 40.105µs I0327 15:09:56.741507 3959 cni.go:403] [openshift-network-diagnostics/network-check-target-xrnlc 3e3bdf0f8ca7d75b5472680a2f337f3fd3ae90e47b9f9c9f8ba81d36581a734b network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-xrnlc 3e3bdf0f8ca7d75b5472680a2f337f3fd3ae90e47b9f9c9f8ba81d36581a734b network default NAD default] I0327 15:09:56.750305 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-xrnlc, hostIfaceName: 3e3bdf0f8ca7d75, network: default, NAD default, SandboxID: "3e3bdf0f8ca7d75b5472680a2f337f3fd3ae90e47b9f9c9f8ba81d36581a734b", PCI device ID: , UID: "42787bcf-e697-4d89-92d6-ab6dcb536749", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0327 15:09:56.792736 3959 cni.go:424] [openshift-network-diagnostics/network-check-target-xrnlc 3e3bdf0f8ca7d75b5472680a2f337f3fd3ae90e47b9f9c9f8ba81d36581a734b network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-xrnlc 3e3bdf0f8ca7d75b5472680a2f337f3fd3ae90e47b9f9c9f8ba81d36581a734b network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:f2:89:22:71:6f\",\"name\":\"3e3bdf0f8ca7d75\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b10ae75f-3b48-4e93-a72c-6a44ab44ba67\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:09:57.382050 3959 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-859d5bccd5-cr7vc] creating logical port openshift-ingress_router-default-859d5bccd5-cr7vc for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:57.382172 3959 kube.go:256] Updating pod openshift-ingress/router-default-859d5bccd5-cr7vc I0327 15:09:57.385107 3959 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-f9877767d-69sb2] creating logical port openshift-ingress_router-default-f9877767d-69sb2 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:57.385188 3959 kube.go:256] Updating pod openshift-ingress/router-default-f9877767d-69sb2 I0327 15:09:57.400972 3959 pod.go:62] [openshift-ingress/router-default-859d5bccd5-cr7vc] pod update took 18.842756ms I0327 15:09:57.400996 3959 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-859d5bccd5-cr7vc] addLogicalPort annotation time took 18.87107ms I0327 15:09:57.402200 3959 pods.go:271] [openshift-ingress/router-default-859d5bccd5-cr7vc] addLogicalPort took 20.172256ms, libovsdb time 897.477µs I0327 15:09:57.403963 3959 pod.go:62] [openshift-ingress/router-default-f9877767d-69sb2] pod update took 18.804273ms I0327 15:09:57.403982 3959 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-f9877767d-69sb2] addLogicalPort annotation time took 18.822465ms I0327 15:09:57.404933 3959 pods.go:271] [openshift-ingress/router-default-f9877767d-69sb2] addLogicalPort took 19.84199ms, libovsdb time 698.033µs I0327 15:09:57.491890 3959 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-8xc4s] creating logical port openshift-console-operator_console-operator-8777bd9bb-8xc4s for pod on switch ip-10-0-1-144.ec2.internal I0327 15:09:57.491990 3959 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-8xc4s I0327 15:09:57.510231 3959 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-8xc4s] pod update took 18.267526ms I0327 15:09:57.510257 3959 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-8xc4s] addLogicalPort annotation time took 18.297195ms I0327 15:09:57.511359 3959 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-8xc4s] addLogicalPort took 19.486213ms, libovsdb time 829.166µs I0327 15:09:57.890225 3959 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-8xc4s 9f3285a556b032a5f6df68cc5f54b83d2077a0a304b5150462d80f831024ff57 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-8xc4s 9f3285a556b032a5f6df68cc5f54b83d2077a0a304b5150462d80f831024ff57 network default NAD default] I0327 15:09:57.898995 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-8xc4s, hostIfaceName: 9f3285a556b032a, network: default, NAD default, SandboxID: "9f3285a556b032a5f6df68cc5f54b83d2077a0a304b5150462d80f831024ff57", PCI device ID: , UID: "542ee515-1edb-4443-ab59-8d58e52f069e", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0327 15:09:57.941840 3959 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-8xc4s 9f3285a556b032a5f6df68cc5f54b83d2077a0a304b5150462d80f831024ff57 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-8xc4s 9f3285a556b032a5f6df68cc5f54b83d2077a0a304b5150462d80f831024ff57 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:6e:e0:65:96:42\",\"name\":\"9f3285a556b032a\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4e9f3329-be78-45c0-ab42-23ad9c95f75a\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:01.931158 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:10:01.955012 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:10:03.386647 3959 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 15:10:03.387555 3959 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 869.321µs I0327 15:10:03.410416 3959 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 15:10:04.306920 3959 namespace.go:98] [openshift-service-ca] adding namespace I0327 15:10:04.307806 3959 namespace.go:102] [openshift-service-ca] adding namespace took 863.745µs I0327 15:10:04.320984 3959 namespace.go:142] [openshift-service-ca] updating namespace I0327 15:10:05.969752 3959 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-6jb47] creating logical port openshift-service-ca_service-ca-f59fc589b-6jb47 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:05.969866 3959 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-6jb47 I0327 15:10:05.983090 3959 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-6jb47] pod update took 13.256772ms I0327 15:10:05.983114 3959 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-6jb47] addLogicalPort annotation time took 13.286556ms I0327 15:10:05.984247 3959 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-6jb47] addLogicalPort took 14.511963ms, libovsdb time 840.931µs I0327 15:10:06.329235 3959 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-6jb47 d466812328e0bfc83ab2b06a9d4c8e648d66bde5f10ee5957298a77717a9620a network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-6jb47 d466812328e0bfc83ab2b06a9d4c8e648d66bde5f10ee5957298a77717a9620a network default NAD default] I0327 15:10:06.337550 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-6jb47, hostIfaceName: d466812328e0bfc, network: default, NAD default, SandboxID: "d466812328e0bfc83ab2b06a9d4c8e648d66bde5f10ee5957298a77717a9620a", PCI device ID: , UID: "ba7ea083-2499-4b6f-ae63-22a23f76c8d6", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0327 15:10:06.378971 3959 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-6jb47 d466812328e0bfc83ab2b06a9d4c8e648d66bde5f10ee5957298a77717a9620a network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-6jb47 d466812328e0bfc83ab2b06a9d4c8e648d66bde5f10ee5957298a77717a9620a network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:de:7f:55:74:6e\",\"name\":\"d466812328e0bfc\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/586a9e96-0216-41a8-9b34-a48b6eceedaf\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:11.286200 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:10:11.286225 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:10:13.785217 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:10:14.988321 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:14.988366 3959 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7c85498c95-q75rf I0327 15:10:14.989727 3959 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7c85498c95-q75rf, ips: 10.128.0.6 I0327 15:10:14.989757 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:15.001384 3959 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7f4b95875b-9lvtg] creating logical port openshift-image-registry_image-registry-7f4b95875b-9lvtg for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:15.001471 3959 kube.go:256] Updating pod openshift-image-registry/image-registry-7f4b95875b-9lvtg I0327 15:10:15.002899 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.002921 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.017760 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c85498c95-q75rf I0327 15:10:15.017802 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c85498c95-q75rf I0327 15:10:15.020716 3959 pod.go:62] [openshift-image-registry/image-registry-7f4b95875b-9lvtg] pod update took 19.269023ms I0327 15:10:15.020741 3959 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7f4b95875b-9lvtg] addLogicalPort annotation time took 19.29922ms I0327 15:10:15.022235 3959 pods.go:271] [openshift-image-registry/image-registry-7f4b95875b-9lvtg] addLogicalPort took 20.866231ms, libovsdb time 1.114253ms I0327 15:10:15.043979 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:15.044561 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:15.056885 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.056903 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.064449 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-59c85fb9b8-2rcd8 I0327 15:10:15.064471 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-59c85fb9b8-2rcd8 I0327 15:10:15.360218 3959 cni.go:403] [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default] I0327 15:10:15.369532 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7f4b95875b-9lvtg, hostIfaceName: 03f7ff8a328d536, network: default, NAD default, SandboxID: "03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935", PCI device ID: , UID: "5f9fe0db-d76e-424f-a014-114e188be9a4", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0327 15:10:15.418050 3959 cni.go:424] [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:e2:d2:4c:9b:dd\",\"name\":\"03f7ff8a328d536\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35a72ba5-c918-4ac1-b5dd-65183024b9a8\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:29.180001 3959 cni.go:403] [openshift-dns/dns-default-mrt9w c7a4df0cabe9d3ad7a4c92201b9275f4cf36dfe2bb62817655557f6ff53bc834 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-mrt9w c7a4df0cabe9d3ad7a4c92201b9275f4cf36dfe2bb62817655557f6ff53bc834 network default NAD default] I0327 15:10:29.192679 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-mrt9w, hostIfaceName: c7a4df0cabe9d3a, network: default, NAD default, SandboxID: "c7a4df0cabe9d3ad7a4c92201b9275f4cf36dfe2bb62817655557f6ff53bc834", PCI device ID: , UID: "f5185368-99f6-4e60-9ddd-21e5431f12df", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0327 15:10:29.240525 3959 cni.go:424] [openshift-dns/dns-default-mrt9w c7a4df0cabe9d3ad7a4c92201b9275f4cf36dfe2bb62817655557f6ff53bc834 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-mrt9w c7a4df0cabe9d3ad7a4c92201b9275f4cf36dfe2bb62817655557f6ff53bc834 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:21:96:ee:5b:fa\",\"name\":\"c7a4df0cabe9d3a\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10f0acec-0c75-4140-b8a7-d1e360f14640\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:29.573630 3959 cni.go:403] [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default] I0327 15:10:29.584437 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-859d5bccd5-cr7vc, hostIfaceName: 4d81af6015e9edd, network: default, NAD default, SandboxID: "4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5", PCI device ID: , UID: "441c469b-f13b-4dee-a1ec-17bc5f40b246", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0327 15:10:29.592580 3959 cni.go:403] [openshift-ingress/router-default-f9877767d-69sb2 88399967110f6fcc7774b3e381551c09b364d833c0341cf11900bc21a8fb882f network default NAD default] ADD starting CNI request [openshift-ingress/router-default-f9877767d-69sb2 88399967110f6fcc7774b3e381551c09b364d833c0341cf11900bc21a8fb882f network default NAD default] I0327 15:10:29.600701 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-f9877767d-69sb2, hostIfaceName: 88399967110f6fc, network: default, NAD default, SandboxID: "88399967110f6fcc7774b3e381551c09b364d833c0341cf11900bc21a8fb882f", PCI device ID: , UID: "a2adf565-4a49-420b-ae7b-8b17493e90ec", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0327 15:10:29.628846 3959 cni.go:424] [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:b2:a7:0b:ff:50\",\"name\":\"4d81af6015e9edd\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/975cde90-13c6-4711-bd64-2e9452dd98a3\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:29.644706 3959 cni.go:424] [openshift-ingress/router-default-f9877767d-69sb2 88399967110f6fcc7774b3e381551c09b364d833c0341cf11900bc21a8fb882f network default NAD default] ADD finished CNI request [openshift-ingress/router-default-f9877767d-69sb2 88399967110f6fcc7774b3e381551c09b364d833c0341cf11900bc21a8fb882f network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:b2:a1:41:8a:07\",\"name\":\"88399967110f6fc\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c7ab1c4-98c3-4279-9d9e-fa84f146533c\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:29.708274 3959 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:10:29.714302 3959 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:29.714316 3959 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:29.714320 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:29.798204 3959 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:10:29.937158 3959 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-8vpgx] creating logical port openshift-insights_insights-runtime-extractor-8vpgx for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:29.937276 3959 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-8vpgx I0327 15:10:29.988428 3959 pod.go:62] [openshift-insights/insights-runtime-extractor-8vpgx] pod update took 51.190357ms I0327 15:10:29.988456 3959 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-8vpgx] addLogicalPort annotation time took 51.225785ms I0327 15:10:29.989718 3959 pods.go:271] [openshift-insights/insights-runtime-extractor-8vpgx] addLogicalPort took 52.603551ms, libovsdb time 880.562µs W0327 15:10:30.080279 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.080298 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.080307 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.084723 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.084738 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.114579 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.114596 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.119277 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.119359 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.119373 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.128511 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.128526 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.128534 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.140028 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.140049 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0327 15:10:30.294829 3959 cni.go:403] [openshift-insights/insights-runtime-extractor-8vpgx ca92848f6ff895abd34fbd9bbe9edb4ebd1eeeb5c1d20423613fd1ab3af52aca network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-8vpgx ca92848f6ff895abd34fbd9bbe9edb4ebd1eeeb5c1d20423613fd1ab3af52aca network default NAD default] I0327 15:10:30.303112 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-8vpgx, hostIfaceName: ca92848f6ff895a, network: default, NAD default, SandboxID: "ca92848f6ff895abd34fbd9bbe9edb4ebd1eeeb5c1d20423613fd1ab3af52aca", PCI device ID: , UID: "4da942b9-106a-4210-93fb-b28695866104", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0327 15:10:30.346520 3959 cni.go:424] [openshift-insights/insights-runtime-extractor-8vpgx ca92848f6ff895abd34fbd9bbe9edb4ebd1eeeb5c1d20423613fd1ab3af52aca network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-8vpgx ca92848f6ff895abd34fbd9bbe9edb4ebd1eeeb5c1d20423613fd1ab3af52aca network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:a9:af:96:50:27\",\"name\":\"ca92848f6ff895a\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/81b9c7a7-e61c-4aa0-ac36-503a4252314a\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:31.072568 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:31.072589 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0327 15:10:31.157633 3959 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0327 15:10:31.170483 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:31.170503 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:31.170511 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0327 15:10:31.272550 3959 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:10:31.356565 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:31.357114 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:31.369128 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:31.369151 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:31.379857 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c85498c95-7sbxq I0327 15:10:31.379875 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c85498c95-7sbxq W0327 15:10:32.050676 3959 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:32.050692 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:32.050697 3959 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:32.348231 3959 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:10:32.938232 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:10:33.082799 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:33.082818 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:33.082824 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:33.082835 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:33.082839 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:33.082846 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:36.113905 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 15:10:37.048921 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:10:37.048943 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:10:42.066507 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066527 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066533 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0327 15:10:42.678330 3959 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j] creating logical port openshift-monitoring_kube-state-metrics-664d8f7fc8-trp2j for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:42.678427 3959 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j I0327 15:10:42.707805 3959 pod.go:62] [openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j] pod update took 29.404673ms I0327 15:10:42.707829 3959 base_network_controller_pods.go:937] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j] addLogicalPort annotation time took 29.43458ms I0327 15:10:42.709208 3959 pods.go:271] [openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j] addLogicalPort took 30.899682ms, libovsdb time 983.318µs I0327 15:10:43.040525 3959 cni.go:403] [openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j 0d2f3967176dba2b0c56b6af0155b89147518cbe4b1895ddf17358ceee979fd0 network default NAD default] ADD starting CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j 0d2f3967176dba2b0c56b6af0155b89147518cbe4b1895ddf17358ceee979fd0 network default NAD default] I0327 15:10:43.048789 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-664d8f7fc8-trp2j, hostIfaceName: 0d2f3967176dba2, network: default, NAD default, SandboxID: "0d2f3967176dba2b0c56b6af0155b89147518cbe4b1895ddf17358ceee979fd0", PCI device ID: , UID: "68d02b21-0977-42aa-86ef-f6f6af672218", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0327 15:10:43.095951 3959 cni.go:424] [openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j 0d2f3967176dba2b0c56b6af0155b89147518cbe4b1895ddf17358ceee979fd0 network default NAD default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-trp2j 0d2f3967176dba2b0c56b6af0155b89147518cbe4b1895ddf17358ceee979fd0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:ff:02:3e:81:88\",\"name\":\"0d2f3967176dba2\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c1051c8-a902-473f-aded-163fc19a0ec8\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:45.651445 3959 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-7bf5fc694-zbnt4] creating logical port openshift-monitoring_thanos-querier-7bf5fc694-zbnt4 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:45.651563 3959 kube.go:256] Updating pod openshift-monitoring/thanos-querier-7bf5fc694-zbnt4 I0327 15:10:45.668735 3959 pod.go:62] [openshift-monitoring/thanos-querier-7bf5fc694-zbnt4] pod update took 17.193455ms I0327 15:10:45.668762 3959 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-7bf5fc694-zbnt4] addLogicalPort annotation time took 17.250413ms I0327 15:10:45.670071 3959 pods.go:271] [openshift-monitoring/thanos-querier-7bf5fc694-zbnt4] addLogicalPort took 18.641819ms, libovsdb time 989.308µs I0327 15:10:46.012910 3959 cni.go:403] [openshift-monitoring/thanos-querier-7bf5fc694-zbnt4 4a9ced186de661f89dc6bda1bfe8da5c8cc2672f13b4206abae826e0e1f29c9c network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-7bf5fc694-zbnt4 4a9ced186de661f89dc6bda1bfe8da5c8cc2672f13b4206abae826e0e1f29c9c network default NAD default] I0327 15:10:46.021621 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-7bf5fc694-zbnt4, hostIfaceName: 4a9ced186de661f, network: default, NAD default, SandboxID: "4a9ced186de661f89dc6bda1bfe8da5c8cc2672f13b4206abae826e0e1f29c9c", PCI device ID: , UID: "b9e1e4cf-9c0f-4324-bcfa-9939beef2c6f", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0327 15:10:46.066432 3959 cni.go:424] [openshift-monitoring/thanos-querier-7bf5fc694-zbnt4 4a9ced186de661f89dc6bda1bfe8da5c8cc2672f13b4206abae826e0e1f29c9c network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-7bf5fc694-zbnt4 4a9ced186de661f89dc6bda1bfe8da5c8cc2672f13b4206abae826e0e1f29c9c network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:c1:9b:7a:de:71\",\"name\":\"4a9ced186de661f\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f76825f-72a0-4697-ada3-0d9b46f5dece\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:46.455200 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:46.455227 3959 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:46.455233 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:10:47.310218 3959 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-dkq4x for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:47.310332 3959 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x I0327 15:10:47.334227 3959 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x] pod update took 23.926074ms I0327 15:10:47.334252 3959 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x] addLogicalPort annotation time took 23.957593ms I0327 15:10:47.335488 3959 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x] addLogicalPort took 25.286138ms, libovsdb time 874.493µs I0327 15:10:47.380544 3959 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-84d869bc95-rthjh] creating logical port openshift-monitoring_metrics-server-84d869bc95-rthjh for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:47.380634 3959 kube.go:256] Updating pod openshift-monitoring/metrics-server-84d869bc95-rthjh I0327 15:10:47.394237 3959 pod.go:62] [openshift-monitoring/metrics-server-84d869bc95-rthjh] pod update took 13.624836ms I0327 15:10:47.394266 3959 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-84d869bc95-rthjh] addLogicalPort annotation time took 13.660134ms I0327 15:10:47.395609 3959 pods.go:271] [openshift-monitoring/metrics-server-84d869bc95-rthjh] addLogicalPort took 15.080187ms, libovsdb time 864.782µs I0327 15:10:47.743814 3959 cni.go:403] [openshift-monitoring/metrics-server-84d869bc95-rthjh 453cce40b34eeccef19301992a0b2e3f8a791b145c95240f13007f0311794cc8 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-84d869bc95-rthjh 453cce40b34eeccef19301992a0b2e3f8a791b145c95240f13007f0311794cc8 network default NAD default] I0327 15:10:47.755596 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-84d869bc95-rthjh, hostIfaceName: 453cce40b34eecc, network: default, NAD default, SandboxID: "453cce40b34eeccef19301992a0b2e3f8a791b145c95240f13007f0311794cc8", PCI device ID: , UID: "a0054540-5977-4334-842c-77daeb431380", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0327 15:10:47.803441 3959 cni.go:424] [openshift-monitoring/metrics-server-84d869bc95-rthjh 453cce40b34eeccef19301992a0b2e3f8a791b145c95240f13007f0311794cc8 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-84d869bc95-rthjh 453cce40b34eeccef19301992a0b2e3f8a791b145c95240f13007f0311794cc8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:48:54:04:2e:e6\",\"name\":\"453cce40b34eecc\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8790615f-f68c-4f2a-8d36-17f9185a9f58\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:48.233600 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:48.233626 3959 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:48.233631 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:10:48.270810 3959 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x 39847acb99edbf9e08dd4441cef95102fbe048c63059d05b8521c97312e6f0e9 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x 39847acb99edbf9e08dd4441cef95102fbe048c63059d05b8521c97312e6f0e9 network default NAD default] I0327 15:10:48.283456 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-dkq4x, hostIfaceName: 39847acb99edbf9, network: default, NAD default, SandboxID: "39847acb99edbf9e08dd4441cef95102fbe048c63059d05b8521c97312e6f0e9", PCI device ID: , UID: "48d35807-1b6d-464d-9bfd-3b82b2b8239e", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0327 15:10:48.334771 3959 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x 39847acb99edbf9e08dd4441cef95102fbe048c63059d05b8521c97312e6f0e9 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-dkq4x 39847acb99edbf9e08dd4441cef95102fbe048c63059d05b8521c97312e6f0e9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:6e:2d:c0:b8:0d\",\"name\":\"39847acb99edbf9\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f70e23d-e603-4063-8c48-fd66d01546bd\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:48.924588 3959 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:48.924755 3959 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0327 15:10:48.955359 3959 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 30.649096ms I0327 15:10:48.955383 3959 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 30.680375ms I0327 15:10:48.957008 3959 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 32.440284ms, libovsdb time 1.258245ms I0327 15:10:49.291389 3959 cni.go:403] [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default] I0327 15:10:49.300457 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 6c3c8dfd06c6b95, network: default, NAD default, SandboxID: "6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5", PCI device ID: , UID: "512adbd0-5719-4afc-bf99-265857cdb9b0", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0327 15:10:49.352703 3959 cni.go:424] [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:c1:06:4d:52:b8\",\"name\":\"6c3c8dfd06c6b95\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2333b934-9d4c-4c2d-9bdb-a0e28045f3ed\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:10:50.496121 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:10:51.151386 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:10:51.200499 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:51.200523 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:51.233516 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.233540 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.233548 3959 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.233554 3959 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.289521 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:51.289543 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:52.161115 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:52.161137 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:52.161142 3959 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:52.161147 3959 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 15:10:53.775131 3959 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-5pd9f] creating logical port openshift-console_downloads-56d45c498b-5pd9f for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:53.775284 3959 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-5pd9f I0327 15:10:53.795280 3959 pod.go:62] [openshift-console/downloads-56d45c498b-5pd9f] pod update took 20.025258ms I0327 15:10:53.795308 3959 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-5pd9f] addLogicalPort annotation time took 20.057135ms I0327 15:10:53.796713 3959 pods.go:271] [openshift-console/downloads-56d45c498b-5pd9f] addLogicalPort took 21.605028ms, libovsdb time 1.052899ms I0327 15:10:54.141440 3959 cni.go:403] [openshift-console/downloads-56d45c498b-5pd9f a9afe10612bdc4f7b8b331b5258606ca005862263a166b1d754b9ccd33146d22 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-5pd9f a9afe10612bdc4f7b8b331b5258606ca005862263a166b1d754b9ccd33146d22 network default NAD default] I0327 15:10:54.150296 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-5pd9f, hostIfaceName: a9afe10612bdc4f, network: default, NAD default, SandboxID: "a9afe10612bdc4f7b8b331b5258606ca005862263a166b1d754b9ccd33146d22", PCI device ID: , UID: "16916bf3-63ad-468e-b08e-a9c259bc79b9", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] W0327 15:10:54.184715 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:54.184796 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:54.184829 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:54.184856 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 15:10:54.200764 3959 cni.go:424] [openshift-console/downloads-56d45c498b-5pd9f a9afe10612bdc4f7b8b331b5258606ca005862263a166b1d754b9ccd33146d22 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-5pd9f a9afe10612bdc4f7b8b331b5258606ca005862263a166b1d754b9ccd33146d22 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:33:59:ad:e2:2a\",\"name\":\"a9afe10612bdc4f\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9c7905fd-15f6-422e-a80b-c38a04f01518\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:54.943762 3959 cni.go:403] [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default] I0327 15:10:55.010920 3959 cni.go:424] [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-7f4b95875b-9lvtg 03f7ff8a328d536add05f778d45ac40ef24c300dbb1bf735f5aa630cea151935 network default NAD default], result "{}", err I0327 15:10:55.056829 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:55.056864 3959 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7f4b95875b-9lvtg I0327 15:10:55.058125 3959 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7f4b95875b-9lvtg, ips: 10.128.0.14 I0327 15:10:55.058154 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:55.067546 3959 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-65b68b76bc-z8rzh] creating logical port openshift-image-registry_image-registry-65b68b76bc-z8rzh for pod on switch ip-10-0-1-144.ec2.internal I0327 15:10:55.067726 3959 kube.go:256] Updating pod openshift-image-registry/image-registry-65b68b76bc-z8rzh I0327 15:10:55.082720 3959 pod.go:62] [openshift-image-registry/image-registry-65b68b76bc-z8rzh] pod update took 15.018953ms I0327 15:10:55.082752 3959 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-65b68b76bc-z8rzh] addLogicalPort annotation time took 15.056193ms I0327 15:10:55.084253 3959 pods.go:271] [openshift-image-registry/image-registry-65b68b76bc-z8rzh] addLogicalPort took 16.722654ms, libovsdb time 1.01923ms I0327 15:10:55.187940 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:55.187968 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:55.198027 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7f4b95875b-9lvtg I0327 15:10:55.198054 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7f4b95875b-9lvtg I0327 15:10:55.444422 3959 cni.go:403] [openshift-image-registry/image-registry-65b68b76bc-z8rzh d21f3411f74da7ce3f6789d00506c91c2c357613afb57394ff7c5b495f9db40e network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-65b68b76bc-z8rzh d21f3411f74da7ce3f6789d00506c91c2c357613afb57394ff7c5b495f9db40e network default NAD default] I0327 15:10:55.454330 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-65b68b76bc-z8rzh, hostIfaceName: d21f3411f74da7c, network: default, NAD default, SandboxID: "d21f3411f74da7ce3f6789d00506c91c2c357613afb57394ff7c5b495f9db40e", PCI device ID: , UID: "1834529e-faf8-4de8-bf2a-a54bb5e09408", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0327 15:10:55.504837 3959 cni.go:424] [openshift-image-registry/image-registry-65b68b76bc-z8rzh d21f3411f74da7ce3f6789d00506c91c2c357613afb57394ff7c5b495f9db40e network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-65b68b76bc-z8rzh d21f3411f74da7ce3f6789d00506c91c2c357613afb57394ff7c5b495f9db40e network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:5d:1c:2f:48:40\",\"name\":\"d21f3411f74da7c\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ece84d67-8c90-4a9d-b312-684e95af4368\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:10:58.364970 3959 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.365545 3959 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.365626 3959 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.365637 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 15:10:58.365656 3959 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.365679 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 15:10:59.395801 3959 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:59.395829 3959 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:59.395847 3959 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:59.395863 3959 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:00.636250 3959 cni.go:403] [openshift-multus/network-metrics-daemon-64l54 a216047f8795eb9d6ba0e461db5770056e82987c5f1ab3beedff6118ad5f59d2 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-64l54 a216047f8795eb9d6ba0e461db5770056e82987c5f1ab3beedff6118ad5f59d2 network default NAD default] I0327 15:11:00.646223 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-64l54, hostIfaceName: a216047f8795eb9, network: default, NAD default, SandboxID: "a216047f8795eb9d6ba0e461db5770056e82987c5f1ab3beedff6118ad5f59d2", PCI device ID: , UID: "7fd9227b-2b92-4cda-bded-56d37a1d0d78", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0327 15:11:00.699173 3959 cni.go:424] [openshift-multus/network-metrics-daemon-64l54 a216047f8795eb9d6ba0e461db5770056e82987c5f1ab3beedff6118ad5f59d2 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-64l54 a216047f8795eb9d6ba0e461db5770056e82987c5f1ab3beedff6118ad5f59d2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:e5:f7:91:c8:e0\",\"name\":\"a216047f8795eb9\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8d2acaab-c1f8-4896-9824-d397e484ffae\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:11:01.196732 3959 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:11:02.881862 3959 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:11:02.891299 3959 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0327 15:11:03.166057 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:11:04.233204 3959 base_network_controller_pods.go:487] [default/openshift-console/console-5dbf498488-265nt] creating logical port openshift-console_console-5dbf498488-265nt for pod on switch ip-10-0-1-144.ec2.internal I0327 15:11:04.233407 3959 kube.go:256] Updating pod openshift-console/console-5dbf498488-265nt I0327 15:11:04.249347 3959 pod.go:62] [openshift-console/console-5dbf498488-265nt] pod update took 15.981494ms I0327 15:11:04.249373 3959 base_network_controller_pods.go:937] [default/openshift-console/console-5dbf498488-265nt] addLogicalPort annotation time took 16.013889ms I0327 15:11:04.250798 3959 pods.go:271] [openshift-console/console-5dbf498488-265nt] addLogicalPort took 17.610604ms, libovsdb time 921.284µs I0327 15:11:09.034324 3959 cni.go:403] [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default] ADD starting CNI request [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default] I0327 15:11:09.043045 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5dbf498488-265nt, hostIfaceName: 236fc764ef06cc2, network: default, NAD default, SandboxID: "236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa", PCI device ID: , UID: "bf036451-b987-4e9e-9a2d-4a76c6f46ba0", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0327 15:11:09.088691 3959 cni.go:424] [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default] ADD finished CNI request [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:50:29:8f:2d:c3\",\"name\":\"236fc764ef06cc2\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49b8ea21-af13-46c2-b679-699f77f28d60\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:11:09.252536 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 15:11:09.529825 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 15:11:09.554216 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 15:11:09.554243 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 15:11:10.285541 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 15:11:11.461362 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:11:11.461393 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:11:12.157211 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:11:12.262298 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:11:12.551285 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:11:12.692379 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:11:13.556765 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0327 15:11:14.619329 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 15:11:14.619359 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 15:11:15.283862 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 15:11:17.199982 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:11:18.686965 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:18.687480 3959 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:19.333821 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:19.333845 3959 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:19.337061 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7f4b95875b-7mwt8 I0327 15:11:19.337082 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7f4b95875b-7mwt8 W0327 15:11:23.604758 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:11:23.638073 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 15:11:39.842673 3959 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-b8htm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:39.843219 3959 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-b8htm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:40.695804 3959 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-b8htm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:40.695834 3959 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-b8htm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:40.702858 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dbf498488-b8htm I0327 15:11:40.702888 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dbf498488-b8htm I0327 15:11:47.264772 3959 cni.go:403] [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default] DEL starting CNI request [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default] I0327 15:11:47.339800 3959 cni.go:424] [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default] DEL finished CNI request [openshift-ingress/router-default-859d5bccd5-cr7vc 4d81af6015e9eddf5493d277d2152cf2870f17331a866c7bde847d5b27bc28e5 network default NAD default], result "{}", err I0327 15:11:47.411280 3959 obj_retry.go:464] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:47.411324 3959 pods.go:176] Deleting pod: openshift-ingress/router-default-859d5bccd5-cr7vc I0327 15:11:47.412645 3959 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-859d5bccd5-cr7vc, ips: 10.128.0.10 I0327 15:11:47.412697 3959 obj_retry.go:464] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:48.408377 3959 obj_retry.go:458] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:48.408408 3959 obj_retry.go:458] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:48.413358 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-859d5bccd5-cr7vc I0327 15:11:48.413392 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-859d5bccd5-cr7vc I0327 15:11:48.584800 3959 cni.go:403] [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default] DEL starting CNI request [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default] I0327 15:11:48.674669 3959 cni.go:424] [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default] DEL finished CNI request [openshift-console/console-5dbf498488-265nt 236fc764ef06cc2b9cad73e5691caad4896f4c85f77fdc4445b808d598cec3aa network default NAD default], result "{}", err I0327 15:11:48.740010 3959 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-265nt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:48.740050 3959 pods.go:176] Deleting pod: openshift-console/console-5dbf498488-265nt I0327 15:11:48.741352 3959 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5dbf498488-265nt, ips: 10.128.0.23 I0327 15:11:48.741375 3959 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-265nt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:49.414181 3959 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-265nt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:49.414211 3959 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-265nt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:49.421343 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dbf498488-265nt I0327 15:11:49.421371 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dbf498488-265nt W0327 15:11:49.470845 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:11:49.470875 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 15:12:04.464866 3959 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:04.465566 3959 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:04.536522 3959 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:04.536551 3959 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:04.542198 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 15:12:04.542224 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 15:12:04.862878 3959 base_network_controller_pods.go:487] [default/openshift-console/console-d7b8685d9-8gfwx] creating logical port openshift-console_console-d7b8685d9-8gfwx for pod on switch ip-10-0-1-144.ec2.internal I0327 15:12:04.863024 3959 kube.go:256] Updating pod openshift-console/console-d7b8685d9-8gfwx I0327 15:12:04.885726 3959 pod.go:62] [openshift-console/console-d7b8685d9-8gfwx] pod update took 22.728212ms I0327 15:12:04.885756 3959 base_network_controller_pods.go:937] [default/openshift-console/console-d7b8685d9-8gfwx] addLogicalPort annotation time took 22.763217ms I0327 15:12:04.887238 3959 pods.go:271] [openshift-console/console-d7b8685d9-8gfwx] addLogicalPort took 24.374995ms, libovsdb time 1.01933ms I0327 15:12:05.227593 3959 cni.go:403] [openshift-console/console-d7b8685d9-8gfwx 01a05111555009cef6cc925b61ee8b65f477417c48e9d93ccc875ed590ff87d2 network default NAD default] ADD starting CNI request [openshift-console/console-d7b8685d9-8gfwx 01a05111555009cef6cc925b61ee8b65f477417c48e9d93ccc875ed590ff87d2 network default NAD default] I0327 15:12:05.235945 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-d7b8685d9-8gfwx, hostIfaceName: 01a05111555009c, network: default, NAD default, SandboxID: "01a05111555009cef6cc925b61ee8b65f477417c48e9d93ccc875ed590ff87d2", PCI device ID: , UID: "531ea05a-8653-403b-89db-7a78fa356199", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0327 15:12:05.282087 3959 cni.go:424] [openshift-console/console-d7b8685d9-8gfwx 01a05111555009cef6cc925b61ee8b65f477417c48e9d93ccc875ed590ff87d2 network default NAD default] ADD finished CNI request [openshift-console/console-d7b8685d9-8gfwx 01a05111555009cef6cc925b61ee8b65f477417c48e9d93ccc875ed590ff87d2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:01:10:ab:ee:ac\",\"name\":\"01a05111555009c\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b42d16e3-6a83-4728-a2e9-6061fc1735dc\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:12:08.750784 3959 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:08.751370 3959 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:09.572740 3959 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:09.572769 3959 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:09.579797 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 15:12:09.579818 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0327 15:12:15.534047 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:12:30.179708 3959 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-c54lq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:30.180247 3959 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-c54lq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:30.913033 3959 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-c54lq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:30.913058 3959 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-c54lq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:30.916397 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d8bc75c77-c54lq I0327 15:12:30.916421 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d8bc75c77-c54lq I0327 15:12:36.281850 3959 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:36.282392 3959 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:36.938841 3959 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:36.938870 3959 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:36.945748 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 15:12:36.945780 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 15:12:40.862825 3959 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-8mglp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:40.863345 3959 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-8mglp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:41.709041 3959 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-8mglp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:41.709066 3959 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-8mglp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:41.711708 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d8bc75c77-8mglp I0327 15:12:41.711732 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d8bc75c77-8mglp I0327 15:13:12.038336 3959 cni.go:403] [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default] I0327 15:13:12.112148 3959 cni.go:424] [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 6c3c8dfd06c6b95813f9df203201642aa44d0ea08e3102da434905badf6e44b5 network default NAD default], result "{}", err I0327 15:13:12.177371 3959 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:13:12.177408 3959 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0327 15:13:12.178801 3959 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.128.0.20 I0327 15:13:12.178828 3959 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:13:12.749325 3959 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:13:12.749367 3959 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:13:12.755592 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 15:13:12.755679 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 15:13:12.794293 3959 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:13:12.794458 3959 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0327 15:13:12.818598 3959 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 24.19164ms I0327 15:13:12.818626 3959 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 24.221389ms I0327 15:13:12.819919 3959 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 25.644798ms, libovsdb time 906.87µs I0327 15:13:13.156446 3959 cni.go:403] [openshift-monitoring/prometheus-k8s-0 30cb8d1544d7f1262e881e47dfe36e5aa254924f6c77842a91ff5abe51a55e78 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 30cb8d1544d7f1262e881e47dfe36e5aa254924f6c77842a91ff5abe51a55e78 network default NAD default] I0327 15:13:13.164703 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 30cb8d1544d7f12, network: default, NAD default, SandboxID: "30cb8d1544d7f1262e881e47dfe36e5aa254924f6c77842a91ff5abe51a55e78", PCI device ID: , UID: "9b227dea-a3bb-4d34-b178-4cc310886918", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0327 15:13:13.208651 3959 cni.go:424] [openshift-monitoring/prometheus-k8s-0 30cb8d1544d7f1262e881e47dfe36e5aa254924f6c77842a91ff5abe51a55e78 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 30cb8d1544d7f1262e881e47dfe36e5aa254924f6c77842a91ff5abe51a55e78 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:49:27:bc:4c:f2\",\"name\":\"30cb8d1544d7f12\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b29dbb0d-3980-4af9-bb69-a0f20262e45f\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:13:59.212457 3959 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0327 15:14:14.038703 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:14:14.038734 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 15:15:01.648520 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0327 15:15:20.666003 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0327 15:15:54.874857 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:15:54.875456 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:15:55.901223 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:15:55.901280 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:01.653193 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0327 15:16:01.678135 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0327 15:16:04.876406 3959 namespace.go:98] [tsf] adding namespace I0327 15:16:04.877286 3959 namespace.go:102] [tsf] adding namespace took 849.563µs I0327 15:16:04.893615 3959 namespace.go:142] [tsf] updating namespace I0327 15:16:04.902558 3959 namespace.go:142] [tsf] updating namespace I0327 15:16:06.671977 3959 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 W0327 15:16:07.804936 3959 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:16:08.847363 3959 namespace.go:98] [konflux-ui] adding namespace I0327 15:16:08.848246 3959 namespace.go:102] [konflux-ui] adding namespace took 859.706µs I0327 15:16:08.851315 3959 namespace.go:98] [tsf-keycloak] adding namespace I0327 15:16:08.851473 3959 namespace.go:98] [tsf-tas] adding namespace I0327 15:16:08.851518 3959 namespace.go:98] [konflux-operator] adding namespace I0327 15:16:08.851626 3959 namespace.go:98] [cert-manager-operator] adding namespace I0327 15:16:08.851714 3959 namespace.go:98] [tsf-tpa] adding namespace I0327 15:16:08.852111 3959 namespace.go:102] [tsf-keycloak] adding namespace took 778.774µs I0327 15:16:08.852687 3959 namespace.go:102] [tsf-tas] adding namespace took 1.199125ms I0327 15:16:08.853214 3959 namespace.go:102] [konflux-operator] adding namespace took 1.639788ms I0327 15:16:08.853676 3959 namespace.go:102] [cert-manager-operator] adding namespace took 2.017705ms I0327 15:16:08.854299 3959 namespace.go:102] [tsf-tpa] adding namespace took 2.574134ms I0327 15:16:08.855703 3959 namespace.go:98] [rhtpa-operator] adding namespace I0327 15:16:08.855770 3959 namespace.go:98] [rhbk-operator] adding namespace I0327 15:16:08.856335 3959 namespace.go:102] [rhtpa-operator] adding namespace took 615.87µs I0327 15:16:08.856988 3959 namespace.go:102] [rhbk-operator] adding namespace took 1.203856ms I0327 15:16:08.868625 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:08.880110 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:08.889200 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:08.901438 3959 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:08.907819 3959 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:08.907833 3959 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:08.909271 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:08.914827 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:08.918482 3959 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:08.924691 3959 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:08.927081 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:08.927226 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:08.941376 3959 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:08.953901 3959 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:08.962307 3959 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:08.974805 3959 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:08.987297 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:08.998015 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:09.005768 3959 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:09.018938 3959 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:09.022065 3959 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:09.039070 3959 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:09.041142 3959 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:09.053714 3959 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:09.894754 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:09.897760 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:10.028939 3959 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-144.ec2.internal I0327 15:16:10.029068 3959 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0327 15:16:10.047571 3959 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 18.53237ms I0327 15:16:10.047597 3959 base_network_controller_pods.go:937] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 18.560482ms I0327 15:16:10.048828 3959 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 19.919233ms, libovsdb time 866.508µs I0327 15:16:10.406887 3959 cni.go:403] [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default] ADD starting CNI request [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default] I0327 15:16:10.416180 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 38fae99cc09c671, network: default, NAD default, SandboxID: "38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454", PCI device ID: , UID: "06e5961a-d808-4da8-8dc5-c1393ec519bd", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0327 15:16:10.463684 3959 cni.go:424] [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default] ADD finished CNI request [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:40:47:60:9a:41\",\"name\":\"38fae99cc09c671\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cb64d706-89cb-4d60-81cf-eec8d13517ea\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:16:16.737439 3959 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.738177 3959 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.738203 3959 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.738210 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:16.738226 3959 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.738231 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:16.758957 3959 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.759538 3959 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.759574 3959 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.759584 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:16.759596 3959 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.759605 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:17.762851 3959 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.762882 3959 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.762900 3959 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.762914 3959 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788161 3959 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788191 3959 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788211 3959 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788221 3959 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:18.840947 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 15:16:18.841008 3959 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4vskp IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4vskp: no pod IPs found I0327 15:16:19.886548 3959 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887142 3959 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887166 3959 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887174 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:19.887184 3959 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887188 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:20.916947 3959 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:20.916973 3959 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:20.916990 3959 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:20.917004 3959 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:21.560151 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=692 I0327 15:16:22.747031 3959 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.747590 3959 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.747624 3959 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.747635 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:22.747647 3959 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.747652 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:23.762743 3959 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.763349 3959 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.763377 3959 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.763385 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:23.763395 3959 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.763399 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:23.783272 3959 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:23.783296 3959 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:23.783307 3959 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:23.783318 3959 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.803527 3959 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.803558 3959 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.803572 3959 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.803584 3959 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:33.838836 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839490 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839522 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839532 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:33.839543 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839547 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... W0327 15:16:34.739001 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 15:16:34.844296 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:34.844323 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:34.844337 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:34.844355 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:16:36.962089 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.184155 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.193097 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.214217 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.214283 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0327 15:16:38.553908 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=78 I0327 15:16:41.254987 3959 namespace.go:98] [cert-manager] adding namespace I0327 15:16:41.255766 3959 namespace.go:102] [cert-manager] adding namespace took 755.838µs I0327 15:16:41.263737 3959 namespace.go:142] [cert-manager] updating namespace I0327 15:16:41.276381 3959 namespace.go:142] [cert-manager] updating namespace I0327 15:16:44.859701 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.860177 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.860200 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.860208 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:44.860219 3959 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.860225 3959 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:45.885141 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:45.885168 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:45.885182 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:45.885191 3959 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:48.642421 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 W0327 15:16:49.049229 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:16:49.049248 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 15:16:50.790315 3959 namespace.go:142] [openshift-operators] updating namespace I0327 15:16:50.811759 3959 namespace.go:142] [openshift-operators] updating namespace I0327 15:16:58.649303 3959 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 15:16:58.650154 3959 namespace.go:102] [trusted-artifact-signer] adding namespace took 829.14µs I0327 15:16:58.673041 3959 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:16:58.718978 3959 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:16:58.999187 3959 namespace.go:142] [openshift-operators] updating namespace W0327 15:16:59.082073 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.515225 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.521145 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.540091 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.545372 3959 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:17:10.638201 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 15:17:11.734145 3959 cni.go:403] [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default] DEL starting CNI request [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default] I0327 15:17:11.805603 3959 cni.go:424] [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default] DEL finished CNI request [tsf/test-tsf-subscriptions 38fae99cc09c6711c4d52c352e6a57b93305ca4a9b6bd649f2d06f98887aa454 network default NAD default], result "{}", err I0327 15:17:11.859686 3959 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:11.859717 3959 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0327 15:17:11.861064 3959 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.26 I0327 15:17:11.861094 3959 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:11.884856 3959 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:11.884891 3959 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:11.896336 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 15:17:11.896359 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 15:17:13.054514 3959 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-1-144.ec2.internal I0327 15:17:13.054630 3959 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0327 15:17:13.070560 3959 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 15.962239ms I0327 15:17:13.070669 3959 base_network_controller_pods.go:937] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 16.062468ms I0327 15:17:13.071961 3959 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 17.465505ms, libovsdb time 950.988µs I0327 15:17:13.428043 3959 cni.go:403] [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default] ADD starting CNI request [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default] I0327 15:17:13.436161 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 7c0ad9b1b727f74, network: default, NAD default, SandboxID: "7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4", PCI device ID: , UID: "841ec807-c123-48b8-84cd-c0ba1a70b61d", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0327 15:17:13.480565 3959 cni.go:424] [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default] ADD finished CNI request [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:d8:e9:04:e1:50\",\"name\":\"7c0ad9b1b727f74\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9721d1c6-a5b2-4cac-871f-00765e06550f\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:14.612200 3959 namespace.go:98] [openshift-pipelines] adding namespace I0327 15:17:14.613132 3959 namespace.go:102] [openshift-pipelines] adding namespace took 906.334µs I0327 15:17:14.633233 3959 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:17:15.333070 3959 namespace.go:142] [cert-manager] updating namespace I0327 15:17:15.525185 3959 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:15.926211 3959 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:15.941746 3959 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-n4kwv for pod on switch ip-10-0-1-144.ec2.internal I0327 15:17:15.941871 3959 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv I0327 15:17:15.955534 3959 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv] pod update took 13.687219ms I0327 15:17:15.955579 3959 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv] addLogicalPort annotation time took 13.734589ms I0327 15:17:15.957098 3959 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv] addLogicalPort took 15.367034ms, libovsdb time 1.063984ms I0327 15:17:16.132039 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:16.330447 3959 namespace.go:142] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] updating namespace I0327 15:17:16.731720 3959 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:16.936754 3959 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:17.133652 3959 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:17.346927 3959 namespace.go:142] [tsf] updating namespace I0327 15:17:17.556608 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:17.764757 3959 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:17.965927 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:18.107719 3959 namespace.go:142] [cert-manager] updating namespace I0327 15:17:18.116286 3959 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:18.126224 3959 namespace.go:142] [default] updating namespace I0327 15:17:18.137492 3959 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:18.148290 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:18.160733 3959 namespace.go:142] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] updating namespace I0327 15:17:18.170900 3959 namespace.go:142] [openshift] updating namespace I0327 15:17:18.179070 3959 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:18.190625 3959 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:18.206162 3959 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:18.208063 3959 namespace.go:142] [tsf] updating namespace I0327 15:17:18.222939 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:18.235389 3959 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:18.255269 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:18.359133 3959 namespace.go:142] [cert-manager] updating namespace I0327 15:17:18.465058 3959 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:18.564253 3959 namespace.go:142] [default] updating namespace I0327 15:17:18.658033 3959 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:18.754773 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:18.857177 3959 namespace.go:142] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] updating namespace I0327 15:17:18.955441 3959 namespace.go:142] [openshift] updating namespace I0327 15:17:19.056590 3959 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:19.160307 3959 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:19.253738 3959 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:19.355058 3959 namespace.go:142] [tsf] updating namespace I0327 15:17:19.456190 3959 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:19.563184 3959 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:19.657231 3959 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:20.009361 3959 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:17:22.625694 3959 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv 3ab36698fc509bc846ab9d5d5aeb68fb2e39d714f250bd3870b4964a0833b8ff network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv 3ab36698fc509bc846ab9d5d5aeb68fb2e39d714f250bd3870b4964a0833b8ff network default NAD default] I0327 15:17:22.634344 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-n4kwv, hostIfaceName: 3ab36698fc509bc, network: default, NAD default, SandboxID: "3ab36698fc509bc846ab9d5d5aeb68fb2e39d714f250bd3870b4964a0833b8ff", PCI device ID: , UID: "55fd8d47-7600-4a11-bd82-487101d9dae7", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0327 15:17:22.678436 3959 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv 3ab36698fc509bc846ab9d5d5aeb68fb2e39d714f250bd3870b4964a0833b8ff network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-n4kwv 3ab36698fc509bc846ab9d5d5aeb68fb2e39d714f250bd3870b4964a0833b8ff network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:74:a6:c0:fe:14\",\"name\":\"3ab36698fc509bc\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0c8fa6c-4816-4830-b632-97ce461ea6df\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:17:23.793096 3959 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0327 15:17:24.636752 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0327 15:17:25.556019 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=105 I0327 15:17:27.769492 3959 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-mgbsk for pod on switch ip-10-0-1-144.ec2.internal I0327 15:17:27.769589 3959 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk I0327 15:17:27.781925 3959 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk] pod update took 12.358582ms I0327 15:17:27.781950 3959 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk] addLogicalPort annotation time took 12.38851ms I0327 15:17:27.783212 3959 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk] addLogicalPort took 13.735527ms, libovsdb time 930.079µs I0327 15:17:28.135081 3959 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk 5ebb3adaf0adca1a8f85cf73d526061a9a8aedd12d6daf2a98b319747d42efe6 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk 5ebb3adaf0adca1a8f85cf73d526061a9a8aedd12d6daf2a98b319747d42efe6 network default NAD default] I0327 15:17:28.144353 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-mgbsk, hostIfaceName: 5ebb3adaf0adca1, network: default, NAD default, SandboxID: "5ebb3adaf0adca1a8f85cf73d526061a9a8aedd12d6daf2a98b319747d42efe6", PCI device ID: , UID: "8620576d-d0db-45f1-8185-46eec3123692", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0327 15:17:28.191655 3959 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk 5ebb3adaf0adca1a8f85cf73d526061a9a8aedd12d6daf2a98b319747d42efe6 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-mgbsk 5ebb3adaf0adca1a8f85cf73d526061a9a8aedd12d6daf2a98b319747d42efe6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:68:81:25:f2:41\",\"name\":\"5ebb3adaf0adca1\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7de30a1d-e473-454e-adb0-9e216c474235\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:17:31.234995 3959 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:31.235012 3959 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:31.235018 3959 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:32.031588 3959 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 15:17:32.256350 3959 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:32.256374 3959 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:32.256380 3959 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:32.256387 3959 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:34.038454 3959 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:17:34.038475 3959 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:17:34.038480 3959 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:17:44.808566 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0327 15:17:49.550330 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 W0327 15:17:55.408970 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 15:17:55.932441 3959 cni.go:403] [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default] DEL starting CNI request [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default] I0327 15:17:56.018367 3959 cni.go:424] [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default] DEL finished CNI request [tsf/test-tsf-infrastructure 7c0ad9b1b727f74984282240f6c94180dd029e0607854b430e024efd74b444c4 network default NAD default], result "{}", err I0327 15:17:56.068170 3959 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:56.068207 3959 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0327 15:17:56.069466 3959 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.27 I0327 15:17:56.069497 3959 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:56.085946 3959 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:56.085973 3959 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:56.091136 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 15:17:56.091162 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 15:17:57.077257 3959 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-1-144.ec2.internal I0327 15:17:57.077348 3959 kube.go:256] Updating pod tsf/test-tsf-iam I0327 15:17:57.092874 3959 pod.go:62] [tsf/test-tsf-iam] pod update took 15.549055ms I0327 15:17:57.092902 3959 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 15.579593ms I0327 15:17:57.094119 3959 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 16.880273ms, libovsdb time 868.488µs I0327 15:17:57.124131 3959 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:17:57.124245 3959 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 15:17:57.142158 3959 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 17.949031ms I0327 15:17:57.142187 3959 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 17.984912ms I0327 15:17:57.143319 3959 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 19.203597ms, libovsdb time 792.058µs I0327 15:17:57.454850 3959 cni.go:403] [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default] I0327 15:17:57.465635 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 67eb2360ad949b5, network: default, NAD default, SandboxID: "67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb", PCI device ID: , UID: "ed978960-9cc4-45f9-b168-98c0852f9188", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0327 15:17:57.483639 3959 cni.go:403] [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default] I0327 15:17:57.494391 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 4b12aaace5b4d8a, network: default, NAD default, SandboxID: "4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809", PCI device ID: , UID: "e5013b4e-67e0-4ff7-b7a8-9324d06c810b", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0327 15:17:57.516118 3959 cni.go:424] [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:25:91:cc:5e:3c\",\"name\":\"67eb2360ad949b5\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f6f5d7f9-ea29-4962-bae0-92ec064a371f\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:57.540577 3959 cni.go:424] [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:a3:47:2b:b2:86\",\"name\":\"4b12aaace5b4d8a\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26792c4a-b4e8-464a-88ed-86dfa9ae1566\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:18:02.971169 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:02.971197 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:02.975277 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 15:18:13.117706 3959 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:18:23.555208 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=119 W0327 15:18:23.797228 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:23.797255 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:23.797628 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 15:18:25.194989 3959 cni.go:403] [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default] I0327 15:18:25.269873 3959 cni.go:424] [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 4b12aaace5b4d8a6e91e9fc79d25337990ea2c9f1ec83016b59d5db3daa85809 network default NAD default], result "{}", err I0327 15:18:25.316086 3959 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:25.316123 3959 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 15:18:25.317477 3959 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.31 I0327 15:18:25.317506 3959 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:26.075332 3959 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:26.075356 3959 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:26.077683 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:18:26.077709 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 15:18:26.106257 3959 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:18:26.106344 3959 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 15:18:26.119776 3959 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 13.458841ms I0327 15:18:26.119793 3959 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 13.47936ms I0327 15:18:26.121024 3959 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 14.785567ms, libovsdb time 922.228µs I0327 15:18:26.463151 3959 cni.go:403] [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default] I0327 15:18:26.471462 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: f604d1ead3efca4, network: default, NAD default, SandboxID: "f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846", PCI device ID: , UID: "605af5c6-52c3-485c-bdff-11d6a16667cd", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0327 15:18:26.516440 3959 cni.go:424] [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:bf:07:1e:91:69\",\"name\":\"f604d1ead3efca4\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ddce3243-fcf4-4acd-b634-19dee672e3f2\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:18:28.097573 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:18:28.097625 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:28.097638 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0327 15:18:30.659461 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0327 15:18:33.721856 3959 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-5rd98 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:18:33.721987 3959 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98 I0327 15:18:33.737142 3959 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98] pod update took 15.180173ms I0327 15:18:33.737165 3959 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98] addLogicalPort annotation time took 15.209406ms I0327 15:18:33.738644 3959 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98] addLogicalPort took 16.805587ms, libovsdb time 1.051622ms I0327 15:18:34.091454 3959 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98 53cf9770e152d4864759e7388d53bede46cd4df643bbd9ca55d97c2e64c8cd61 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98 53cf9770e152d4864759e7388d53bede46cd4df643bbd9ca55d97c2e64c8cd61 network default NAD default] I0327 15:18:34.103900 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-5rd98, hostIfaceName: 53cf9770e152d48, network: default, NAD default, SandboxID: "53cf9770e152d4864759e7388d53bede46cd4df643bbd9ca55d97c2e64c8cd61", PCI device ID: , UID: "4606da47-444a-4043-b923-566ad8b88ab3", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0327 15:18:34.154125 3959 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98 53cf9770e152d4864759e7388d53bede46cd4df643bbd9ca55d97c2e64c8cd61 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-5rd98 53cf9770e152d4864759e7388d53bede46cd4df643bbd9ca55d97c2e64c8cd61 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:0a:a9:db:ff:9a\",\"name\":\"53cf9770e152d48\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f9389a9-0dd0-4f54-9753-a5dfa152674b\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:18:39.161515 3959 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:18:39.161534 3959 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:18:39.161539 3959 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 15:18:41.554771 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=225 W0327 15:18:41.797084 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:41.797145 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:41.797282 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 15:18:42.678402 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 W0327 15:18:44.198215 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:18:44.198235 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:18:44.198241 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 15:18:47.362552 3959 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-29nwp] creating logical port tsf-keycloak_tsf-iam-29nwp for pod on switch ip-10-0-1-144.ec2.internal I0327 15:18:47.362700 3959 kube.go:256] Updating pod tsf-keycloak/tsf-iam-29nwp I0327 15:18:47.375963 3959 pod.go:62] [tsf-keycloak/tsf-iam-29nwp] pod update took 13.306616ms I0327 15:18:47.375984 3959 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-29nwp] addLogicalPort annotation time took 13.333371ms I0327 15:18:47.377228 3959 pods.go:271] [tsf-keycloak/tsf-iam-29nwp] addLogicalPort took 14.69461ms, libovsdb time 863.584µs I0327 15:18:47.720891 3959 cni.go:403] [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default] I0327 15:18:47.729033 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-29nwp, hostIfaceName: af56c621e145af0, network: default, NAD default, SandboxID: "af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1", PCI device ID: , UID: "f259f38d-2dad-4cf5-b7ce-8727e9d16192", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0327 15:18:47.774229 3959 cni.go:424] [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:e6:4c:83:cd:98\",\"name\":\"af56c621e145af0\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a19e2d3f-861c-4803-8650-a3609d6fa3ca\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:18:58.271023 3959 cni.go:403] [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default] I0327 15:18:58.354676 3959 cni.go:424] [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 67eb2360ad949b52d5f1a652a25c5a870730f9a5e797966758751160739b80cb network default NAD default], result "{}", err I0327 15:18:58.405150 3959 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:58.405198 3959 pods.go:176] Deleting pod: tsf/test-tsf-iam I0327 15:18:58.406510 3959 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.30 I0327 15:18:58.406536 3959 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:59.010955 3959 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:18:59.011075 3959 egressfirewall.go:628] Syncing node "ip-10-0-2-34.ec2.internal" for egress firewall I0327 15:18:59.011082 3959 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 135.159µs I0327 15:19:05.306779 3959 cni.go:403] [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default] I0327 15:19:05.387589 3959 cni.go:424] [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-29nwp af56c621e145af046859cc4fc76a3f66a320953d1b7fd6d8092a3d257f7fc8c1 network default NAD default], result "{}", err I0327 15:19:05.438198 3959 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-29nwp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:05.438233 3959 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-29nwp I0327 15:19:05.439510 3959 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-29nwp, ips: 10.128.0.34 I0327 15:19:05.439535 3959 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-29nwp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:06.469598 3959 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-29nwp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:06.469620 3959 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-29nwp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:06.699074 3959 cni.go:403] [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default] I0327 15:19:06.766878 3959 cni.go:424] [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 f604d1ead3efca46be233dd09a7bb0c823426b4ce36ccb31e87dfdbe45a3e846 network default NAD default], result "{}", err I0327 15:19:06.814041 3959 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:06.814079 3959 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 15:19:06.815412 3959 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.32 I0327 15:19:06.815438 3959 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:07.309711 3959 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:07.309735 3959 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:07.315240 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:19:07.315262 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 15:19:07.347782 3959 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:19:07.347873 3959 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 15:19:07.363166 3959 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 15.317232ms I0327 15:19:07.363192 3959 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 15.347077ms I0327 15:19:07.364432 3959 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 16.66484ms, libovsdb time 874.274µs I0327 15:19:07.707308 3959 cni.go:403] [tsf-keycloak/keycloak-0 2ac730f3bd14414d506482555546bf78368f7762bbf84cd66d34fb1be3d6a30b network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 2ac730f3bd14414d506482555546bf78368f7762bbf84cd66d34fb1be3d6a30b network default NAD default] I0327 15:19:07.715680 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 2ac730f3bd14414, network: default, NAD default, SandboxID: "2ac730f3bd14414d506482555546bf78368f7762bbf84cd66d34fb1be3d6a30b", PCI device ID: , UID: "34b0bf0f-90af-4fe3-b114-df3fa014274c", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0327 15:19:07.761098 3959 cni.go:424] [tsf-keycloak/keycloak-0 2ac730f3bd14414d506482555546bf78368f7762bbf84cd66d34fb1be3d6a30b network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 2ac730f3bd14414d506482555546bf78368f7762bbf84cd66d34fb1be3d6a30b network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:9e:2d:ac:54:54\",\"name\":\"2ac730f3bd14414\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7b82d7b-c4bc-4dd4-b1d5-34a6cfaef848\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:19:08.166011 3959 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-ghzpm for pod on switch ip-10-0-1-144.ec2.internal I0327 15:19:08.166105 3959 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm I0327 15:19:08.180515 3959 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm] pod update took 14.432415ms I0327 15:19:08.180546 3959 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm] addLogicalPort annotation time took 14.467196ms I0327 15:19:08.181807 3959 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm] addLogicalPort took 15.811435ms, libovsdb time 811.405µs I0327 15:19:08.354472 3959 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-fqj7c for pod on switch ip-10-0-1-144.ec2.internal I0327 15:19:08.354572 3959 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c I0327 15:19:08.365949 3959 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c] pod update took 11.398122ms I0327 15:19:08.365975 3959 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c] addLogicalPort annotation time took 11.436553ms I0327 15:19:08.367243 3959 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c] addLogicalPort took 12.791383ms, libovsdb time 927.693µs I0327 15:19:08.527629 3959 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm 8ce4fb83f2301bf70e51c09f99fc9582be64eace1ca64fbf3a4670c3bd90024e network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm 8ce4fb83f2301bf70e51c09f99fc9582be64eace1ca64fbf3a4670c3bd90024e network default NAD default] I0327 15:19:08.535976 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-ghzpm, hostIfaceName: 8ce4fb83f2301bf, network: default, NAD default, SandboxID: "8ce4fb83f2301bf70e51c09f99fc9582be64eace1ca64fbf3a4670c3bd90024e", PCI device ID: , UID: "cafcf9cd-3a55-40cd-a053-494585ac6c23", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0327 15:19:08.713405 3959 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c da7d5b0e910c2e6df3252c800ac32a2afdece65f4908147c6a7b61da1f45a935 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c da7d5b0e910c2e6df3252c800ac32a2afdece65f4908147c6a7b61da1f45a935 network default NAD default] I0327 15:19:08.723614 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-fqj7c, hostIfaceName: da7d5b0e910c2e6, network: default, NAD default, SandboxID: "da7d5b0e910c2e6df3252c800ac32a2afdece65f4908147c6a7b61da1f45a935", PCI device ID: , UID: "c152b2a9-289d-444b-bdb5-b9f9e291f544", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0327 15:19:08.765371 3959 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c da7d5b0e910c2e6df3252c800ac32a2afdece65f4908147c6a7b61da1f45a935 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-fqj7c da7d5b0e910c2e6df3252c800ac32a2afdece65f4908147c6a7b61da1f45a935 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:03:fb:78:44:1a\",\"name\":\"da7d5b0e910c2e6\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ec865c3-5572-4e4b-8b95-b3f6ee9a0921\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:19:08.786942 3959 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm 8ce4fb83f2301bf70e51c09f99fc9582be64eace1ca64fbf3a4670c3bd90024e network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-ghzpm 8ce4fb83f2301bf70e51c09f99fc9582be64eace1ca64fbf3a4670c3bd90024e network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:90:01:2c:48:70\",\"name\":\"8ce4fb83f2301bf\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/614677ea-42a3-4361-b6e5-58118e7ee60a\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:19:09.331750 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:09.331781 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:09.331856 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:11.345385 3959 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 15:19:11.345418 3959 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 15:19:11.753308 3959 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0327 15:19:12.357888 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 15:19:12.357919 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0327 15:19:21.963215 3959 namespace.go:142] [openshift-pipelines] updating namespace W0327 15:19:23.035262 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:23.038245 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:23.038268 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0327 15:19:23.836557 3959 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-4ndgv for pod on switch ip-10-0-1-144.ec2.internal I0327 15:19:23.836690 3959 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv I0327 15:19:23.850038 3959 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv] pod update took 13.40379ms I0327 15:19:23.850065 3959 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv] addLogicalPort annotation time took 13.433694ms I0327 15:19:23.851650 3959 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv] addLogicalPort took 15.112681ms, libovsdb time 1.023377ms I0327 15:19:24.198128 3959 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv 3d949018414ab0967456ab878815bcc6b7268c24c48cbcb0b888018c9d61eb29 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv 3d949018414ab0967456ab878815bcc6b7268c24c48cbcb0b888018c9d61eb29 network default NAD default] I0327 15:19:24.207409 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-4ndgv, hostIfaceName: 3d949018414ab09, network: default, NAD default, SandboxID: "3d949018414ab0967456ab878815bcc6b7268c24c48cbcb0b888018c9d61eb29", PCI device ID: , UID: "f0ee9f28-dca2-4083-b439-1e0855c46b8f", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0327 15:19:24.251988 3959 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv 3d949018414ab0967456ab878815bcc6b7268c24c48cbcb0b888018c9d61eb29 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-4ndgv 3d949018414ab0967456ab878815bcc6b7268c24c48cbcb0b888018c9d61eb29 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:89:87:11:63:38\",\"name\":\"3d949018414ab09\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4f94dd6-3de9-49ac-92ee-c57cb7cbb6b4\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:19:58.487493 3959 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:58.487519 3959 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:58.493724 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:19:58.493747 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:19:58.613690 3959 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-1-144.ec2.internal I0327 15:19:58.613816 3959 kube.go:256] Updating pod tsf/test-tsf-iam I0327 15:19:58.626855 3959 pod.go:62] [tsf/test-tsf-iam] pod update took 13.063253ms I0327 15:19:58.626877 3959 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 13.089416ms I0327 15:19:58.628308 3959 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 14.660668ms, libovsdb time 1.027108ms I0327 15:19:58.973689 3959 cni.go:403] [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default] I0327 15:19:58.982086 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 02757515e807416, network: default, NAD default, SandboxID: "02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15", PCI device ID: , UID: "4b08e8be-f9ce-40bf-8198-13e29812e5e7", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0327 15:19:59.027114 3959 cni.go:424] [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:10:4e:d8:b4:d0\",\"name\":\"02757515e807416\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c2dc355-28ee-463b-ae20-39261ad80a1f\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:20:04.141952 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:04.142528 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:05.024775 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:05.024799 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:07.627314 3959 cni.go:403] [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default] I0327 15:20:07.713409 3959 cni.go:424] [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 02757515e807416b72cc37850a0a95ef8b5a35a60daf85f40749599bd662ee15 network default NAD default], result "{}", err I0327 15:20:07.766259 3959 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:07.766297 3959 pods.go:176] Deleting pod: tsf/test-tsf-iam I0327 15:20:07.767469 3959 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.39 I0327 15:20:07.767496 3959 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:07.786897 3959 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:07.786922 3959 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:07.792467 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:20:07.792496 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:08.530151 3959 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-1-144.ec2.internal I0327 15:20:08.530244 3959 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0327 15:20:08.547566 3959 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 17.345461ms I0327 15:20:08.547591 3959 base_network_controller_pods.go:937] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 17.378595ms I0327 15:20:08.548793 3959 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 18.659467ms, libovsdb time 847.945µs I0327 15:20:08.887682 3959 cni.go:403] [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default] ADD starting CNI request [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default] I0327 15:20:08.895947 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 3fcaaf7d08f5424, network: default, NAD default, SandboxID: "3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5", PCI device ID: , UID: "af975919-c34d-4805-81d0-56e54b004dc3", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0327 15:20:08.939973 3959 cni.go:424] [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default] ADD finished CNI request [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:3c:ad:e1:27:af\",\"name\":\"3fcaaf7d08f5424\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55ada162-102d-4755-a401-04413695fc42\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:20:10.445416 3959 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-d5ff4fb55-v6hrd] creating logical port tsf-tas_fulcio-server-d5ff4fb55-v6hrd for pod on switch ip-10-0-1-144.ec2.internal I0327 15:20:10.445502 3959 kube.go:256] Updating pod tsf-tas/fulcio-server-d5ff4fb55-v6hrd I0327 15:20:10.462649 3959 pod.go:62] [tsf-tas/fulcio-server-d5ff4fb55-v6hrd] pod update took 17.169077ms I0327 15:20:10.462689 3959 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-d5ff4fb55-v6hrd] addLogicalPort annotation time took 17.213203ms I0327 15:20:10.463798 3959 pods.go:271] [tsf-tas/fulcio-server-d5ff4fb55-v6hrd] addLogicalPort took 18.39661ms, libovsdb time 735.429µs I0327 15:20:10.801725 3959 cni.go:403] [tsf-tas/fulcio-server-d5ff4fb55-v6hrd d322587a335b5d8bd3a47d7bdab5479c09d7bf85702ecc9557b0f9aca5d1dcd9 network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-d5ff4fb55-v6hrd d322587a335b5d8bd3a47d7bdab5479c09d7bf85702ecc9557b0f9aca5d1dcd9 network default NAD default] I0327 15:20:10.810143 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-d5ff4fb55-v6hrd, hostIfaceName: d322587a335b5d8, network: default, NAD default, SandboxID: "d322587a335b5d8bd3a47d7bdab5479c09d7bf85702ecc9557b0f9aca5d1dcd9", PCI device ID: , UID: "fd2cb8e3-3f9e-46d5-a3a8-bd98eb2d4475", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0327 15:20:10.856803 3959 cni.go:424] [tsf-tas/fulcio-server-d5ff4fb55-v6hrd d322587a335b5d8bd3a47d7bdab5479c09d7bf85702ecc9557b0f9aca5d1dcd9 network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-d5ff4fb55-v6hrd d322587a335b5d8bd3a47d7bdab5479c09d7bf85702ecc9557b0f9aca5d1dcd9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:c3:45:37:08:8a\",\"name\":\"d322587a335b5d8\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f4decef-7ae7-438d-b14d-debc5bb20974\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:20:13.674469 3959 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:13.674488 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:13.711708 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:13.711730 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:17.967034 3959 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 15:20:17.990240 3959 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0327 15:20:22.252272 3959 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:22.252940 3959 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:23.272818 3959 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:23.272841 3959 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:20:26.140741 3959 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 15:20:56.871428 3959 cni.go:403] [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default] DEL starting CNI request [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default] I0327 15:20:56.948021 3959 cni.go:424] [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default] DEL finished CNI request [tsf-tas/test-tsf-tas 3fcaaf7d08f542494eb53bdf164d6df1e3db0a7db0091d51a86feb3a0a91aaf5 network default NAD default], result "{}", err I0327 15:20:57.010600 3959 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:57.010628 3959 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0327 15:20:57.011844 3959 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.40 I0327 15:20:57.011871 3959 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:57.030600 3959 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:57.030623 3959 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:57.037139 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 15:20:57.037157 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 15:21:05.477575 3959 obj_retry.go:464] Detected object tsf/patch-tekton-config-56jxt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:05.478125 3959 obj_retry.go:464] Detected object tsf/patch-tekton-config-56jxt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:06.498040 3959 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:06.498067 3959 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:06.593995 3959 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-wn9mw] creating logical port tsf_tsf-tekton-configuration-wn9mw for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:06.594096 3959 kube.go:256] Updating pod tsf/tsf-tekton-configuration-wn9mw I0327 15:21:06.608127 3959 pod.go:62] [tsf/tsf-tekton-configuration-wn9mw] pod update took 14.058877ms I0327 15:21:06.608153 3959 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-wn9mw] addLogicalPort annotation time took 14.089774ms I0327 15:21:06.609508 3959 pods.go:271] [tsf/tsf-tekton-configuration-wn9mw] addLogicalPort took 15.527095ms, libovsdb time 950.148µs I0327 15:21:06.951622 3959 cni.go:403] [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default] I0327 15:21:06.960117 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-wn9mw, hostIfaceName: 02940aac73eedf1, network: default, NAD default, SandboxID: "02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919", PCI device ID: , UID: "4e8a3ee2-923f-4829-a847-2a312b5d764a", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0327 15:21:07.004277 3959 cni.go:424] [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:cf:7c:70:e7:43\",\"name\":\"02940aac73eedf1\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/94674408-fb5f-48a5-8c1e-a633d6ec8981\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:08.397736 3959 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:08.398425 3959 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:09.271676 3959 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:09.271703 3959 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:10.511454 3959 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:10.512029 3959 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:11.483404 3959 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:11.483429 3959 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:11.674186 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 15:21:11.943637 3959 cni.go:403] [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default] I0327 15:21:12.023452 3959 cni.go:424] [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-wn9mw 02940aac73eedf100c7ea4dd58693175d3aedd6fc0bc37ad0540bdb11e678919 network default NAD default], result "{}", err I0327 15:21:12.073328 3959 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-wn9mw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:12.073353 3959 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-wn9mw I0327 15:21:12.074772 3959 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-wn9mw, ips: 10.128.0.42 I0327 15:21:12.074806 3959 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-wn9mw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:12.111546 3959 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-77895cb45d-ptk8k] creating logical port tsf-tas_rekor-search-ui-77895cb45d-ptk8k for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:12.111652 3959 kube.go:256] Updating pod tsf-tas/rekor-search-ui-77895cb45d-ptk8k I0327 15:21:12.126422 3959 pod.go:62] [tsf-tas/rekor-search-ui-77895cb45d-ptk8k] pod update took 14.802937ms I0327 15:21:12.126444 3959 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-77895cb45d-ptk8k] addLogicalPort annotation time took 14.831006ms I0327 15:21:12.127811 3959 pods.go:271] [tsf-tas/rekor-search-ui-77895cb45d-ptk8k] addLogicalPort took 16.292181ms, libovsdb time 971.258µs I0327 15:21:12.477573 3959 cni.go:403] [tsf-tas/rekor-search-ui-77895cb45d-ptk8k 63810773703a1a9b7604b642076c28cb8fdbf76e9d977fc0f1aa0e9b618f1fd9 network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-77895cb45d-ptk8k 63810773703a1a9b7604b642076c28cb8fdbf76e9d977fc0f1aa0e9b618f1fd9 network default NAD default] I0327 15:21:12.485746 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-77895cb45d-ptk8k, hostIfaceName: 63810773703a1a9, network: default, NAD default, SandboxID: "63810773703a1a9b7604b642076c28cb8fdbf76e9d977fc0f1aa0e9b618f1fd9", PCI device ID: , UID: "d3024ded-f06f-425c-989a-e94f73eba497", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0327 15:21:12.529716 3959 cni.go:424] [tsf-tas/rekor-search-ui-77895cb45d-ptk8k 63810773703a1a9b7604b642076c28cb8fdbf76e9d977fc0f1aa0e9b618f1fd9 network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-77895cb45d-ptk8k 63810773703a1a9b7604b642076c28cb8fdbf76e9d977fc0f1aa0e9b618f1fd9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:ab:af:14:e0:d4\",\"name\":\"63810773703a1a9\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4a30e86b-36fb-4ac6-8090-c3f072e920d3\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:13.093151 3959 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-wn9mw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.093173 3959 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-wn9mw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.143580 3959 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.143606 3959 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.147683 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-56jxt I0327 15:21:13.147697 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-56jxt W0327 15:21:14.404773 3959 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0327 15:21:15.966463 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0327 15:21:18.318049 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0327 15:21:24.605489 3959 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:24.606132 3959 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:24.621110 3959 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:24.621128 3959 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:24.626324 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 15:21:24.626346 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 15:21:25.162203 3959 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:25.162310 3959 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0327 15:21:25.174832 3959 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 12.55144ms I0327 15:21:25.174855 3959 base_network_controller_pods.go:937] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 12.581485ms I0327 15:21:25.176111 3959 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 13.928448ms, libovsdb time 855.579µs I0327 15:21:25.521157 3959 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default] I0327 15:21:25.529323 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 25ac858bccaa932, network: default, NAD default, SandboxID: "25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a", PCI device ID: , UID: "eeac67c9-41b4-4ef6-a79f-769b37cbe541", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0327 15:21:25.574575 3959 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:ba:fe:0f:d3:9b\",\"name\":\"25ac858bccaa932\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cdc64627-412c-4074-94ee-b30c49052de1\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:21:26.482513 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0327 15:21:27.976772 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0327 15:21:29.032798 3959 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default] I0327 15:21:29.108752 3959 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 25ac858bccaa9323b2d1d019b9bd0704ba28d1f8cf6d91f2a81b07eb5410353a network default NAD default], result "{}", err I0327 15:21:29.158153 3959 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:29.158187 3959 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0327 15:21:29.159573 3959 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.44 I0327 15:21:29.159603 3959 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:29.178167 3959 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:29.178195 3959 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:29.183823 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 15:21:29.183847 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 15:21:29.376902 3959 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:29.377024 3959 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0327 15:21:29.389226 3959 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 12.228957ms I0327 15:21:29.389246 3959 base_network_controller_pods.go:937] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 12.252964ms I0327 15:21:29.390470 3959 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 13.58421ms, libovsdb time 844.346µs I0327 15:21:29.733094 3959 cni.go:403] [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default] ADD starting CNI request [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default] I0327 15:21:29.741264 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 4e08651f1956731, network: default, NAD default, SandboxID: "4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8", PCI device ID: , UID: "f752805f-b028-49fd-a43a-8b447a2f3610", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0327 15:21:29.786125 3959 cni.go:424] [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:f0:12:92:2d:8f\",\"name\":\"4e08651f1956731\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/504844ca-37af-481e-a6fa-87f06d856d05\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:30.537770 3959 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-rmnrb] creating logical port tsf-tpa_create-db-rmnrb for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:30.537855 3959 kube.go:256] Updating pod tsf-tpa/create-db-rmnrb I0327 15:21:30.550582 3959 pod.go:62] [tsf-tpa/create-db-rmnrb] pod update took 12.747199ms I0327 15:21:30.550601 3959 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-rmnrb] addLogicalPort annotation time took 12.773168ms I0327 15:21:30.551868 3959 pods.go:271] [tsf-tpa/create-db-rmnrb] addLogicalPort took 14.11256ms, libovsdb time 900.479µs I0327 15:21:30.681053 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0327 15:21:30.934290 3959 cni.go:403] [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default] ADD starting CNI request [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default] I0327 15:21:30.943161 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-rmnrb, hostIfaceName: 51ab6935893c6bd, network: default, NAD default, SandboxID: "51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26", PCI device ID: , UID: "3562bff4-f3d1-4133-bf89-4caa8940301b", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0327 15:21:30.986421 3959 cni.go:424] [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default] ADD finished CNI request [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:ae:86:dd:e9:8b\",\"name\":\"51ab6935893c6bd\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6effcf69-0aec-45ab-ae51-621c3b7400b0\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:35.075985 3959 cni.go:403] [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default] DEL starting CNI request [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default] I0327 15:21:35.156400 3959 cni.go:424] [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default] DEL finished CNI request [tsf-tpa/create-db-rmnrb 51ab6935893c6bd5160dc3cc4daffe130d0f593a4c30ec4c77e743927fbcbe26 network default NAD default], result "{}", err I0327 15:21:35.211309 3959 obj_retry.go:464] Detected object tsf-tpa/create-db-rmnrb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:35.211346 3959 pods.go:176] Deleting pod: tsf-tpa/create-db-rmnrb I0327 15:21:35.212734 3959 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-rmnrb, ips: 10.128.0.46 I0327 15:21:35.212764 3959 obj_retry.go:464] Detected object tsf-tpa/create-db-rmnrb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:36.229823 3959 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:36.229855 3959 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:41.717390 3959 obj_retry.go:464] Detected object tsf-tpa/migrate-db-fwvnf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:41.717956 3959 obj_retry.go:464] Detected object tsf-tpa/migrate-db-fwvnf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:42.589403 3959 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.589427 3959 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:43.323614 3959 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-7fzwb-wm8mk] creating logical port tsf-tas_tuf-repository-init-7fzwb-wm8mk for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:43.323738 3959 kube.go:256] Updating pod tsf-tas/tuf-repository-init-7fzwb-wm8mk I0327 15:21:43.335550 3959 pod.go:62] [tsf-tas/tuf-repository-init-7fzwb-wm8mk] pod update took 11.84922ms I0327 15:21:43.335570 3959 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-7fzwb-wm8mk] addLogicalPort annotation time took 11.874554ms I0327 15:21:43.336892 3959 pods.go:271] [tsf-tas/tuf-repository-init-7fzwb-wm8mk] addLogicalPort took 13.29405ms, libovsdb time 904.078µs I0327 15:21:44.732737 3959 obj_retry.go:464] Detected object tsf-tpa/create-importers-lnpnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:44.733291 3959 obj_retry.go:464] Detected object tsf-tpa/create-importers-lnpnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:45.758829 3959 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:45.758846 3959 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:47.886716 3959 cni.go:403] [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default] I0327 15:21:47.895256 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-7fzwb-wm8mk, hostIfaceName: 3d23ceae15f058a, network: default, NAD default, SandboxID: "3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e", PCI device ID: , UID: "1ef38d67-8af7-4253-bc1c-094559ba30fc", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0327 15:21:47.941300 3959 cni.go:424] [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:34:83:a3:fb:1f\",\"name\":\"3d23ceae15f058a\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65bafbc1-ee5c-4d71-a565-0906b196f72a\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:52.168521 3959 cni.go:403] [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default] I0327 15:21:52.250495 3959 cni.go:424] [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-7fzwb-wm8mk 3d23ceae15f058a94bf9c82a6e45d518260e9422f2685606f75135db52fc907e network default NAD default], result "{}", err I0327 15:21:52.301265 3959 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:52.301303 3959 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-7fzwb-wm8mk I0327 15:21:52.302739 3959 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-7fzwb-wm8mk, ips: 10.128.0.47 I0327 15:21:52.302771 3959 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:53.182030 3959 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:53.182051 3959 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:21:54.560043 3959 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0327 15:21:55.408957 3959 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-65w4s] creating logical port tsf-tas_tuf-bc5768c88-65w4s for pod on switch ip-10-0-1-144.ec2.internal I0327 15:21:55.409077 3959 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-65w4s I0327 15:21:55.422012 3959 pod.go:62] [tsf-tas/tuf-bc5768c88-65w4s] pod update took 12.95768ms I0327 15:21:55.422034 3959 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-65w4s] addLogicalPort annotation time took 12.983727ms I0327 15:21:55.423477 3959 pods.go:271] [tsf-tas/tuf-bc5768c88-65w4s] addLogicalPort took 14.545259ms, libovsdb time 946.989µs I0327 15:21:57.198214 3959 cni.go:403] [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default] DEL starting CNI request [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default] I0327 15:21:57.272558 3959 cni.go:424] [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 4e08651f195673168057e076b8d37c962ce9cac8a4a339f9445ef7f9ec21fdd8 network default NAD default], result "{}", err I0327 15:21:57.322860 3959 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:57.322912 3959 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0327 15:21:57.324514 3959 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.45 I0327 15:21:57.324546 3959 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:57.342234 3959 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:57.342254 3959 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:57.353823 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 15:21:57.353850 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 15:21:57.867052 3959 cni.go:403] [tsf-tas/tuf-bc5768c88-65w4s eeddcd34ba4820fec7d173a6ceece2df9bdaee1a9d1f62a84e5610b1258dac89 network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-65w4s eeddcd34ba4820fec7d173a6ceece2df9bdaee1a9d1f62a84e5610b1258dac89 network default NAD default] I0327 15:21:57.875944 3959 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-65w4s, hostIfaceName: eeddcd34ba4820f, network: default, NAD default, SandboxID: "eeddcd34ba4820fec7d173a6ceece2df9bdaee1a9d1f62a84e5610b1258dac89", PCI device ID: , UID: "7d39d522-5b9e-403d-82fb-724cd3ec714a", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0327 15:21:57.921390 3959 cni.go:424] [tsf-tas/tuf-bc5768c88-65w4s eeddcd34ba4820fec7d173a6ceece2df9bdaee1a9d1f62a84e5610b1258dac89 network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-65w4s eeddcd34ba4820fec7d173a6ceece2df9bdaee1a9d1f62a84e5610b1258dac89 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:04:88:75:ac:f6\",\"name\":\"eeddcd34ba4820f\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f02dba6-6cca-4cab-99df-28f617735e07\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:58.078476 3959 namespace.go:98] [build-service] adding namespace I0327 15:21:58.078596 3959 namespace.go:98] [image-controller] adding namespace I0327 15:21:58.079400 3959 namespace.go:102] [build-service] adding namespace took 902.342µs I0327 15:21:58.079884 3959 namespace.go:98] [integration-service] adding namespace I0327 15:21:58.080044 3959 namespace.go:102] [image-controller] adding namespace took 1.429809ms I0327 15:21:58.080511 3959 namespace.go:102] [integration-service] adding namespace took 615.18µs I0327 15:21:58.123181 3959 namespace.go:142] [build-service] updating namespace I0327 15:21:58.142938 3959 namespace.go:142] [build-service] updating namespace I0327 15:21:58.152055 3959 namespace.go:142] [image-controller] updating namespace I0327 15:21:58.168981 3959 namespace.go:142] [image-controller] updating namespace I0327 15:21:58.174774 3959 namespace.go:142] [integration-service] updating namespace I0327 15:21:58.204532 3959 namespace.go:142] [integration-service] updating namespace I0327 15:21:58.280455 3959 namespace.go:142] [build-service] updating namespace I0327 15:21:58.305219 3959 namespace.go:142] [integration-service] updating namespace I0327 15:21:58.329173 3959 namespace.go:98] [release-service] adding namespace I0327 15:21:58.330041 3959 namespace.go:102] [release-service] adding namespace took 851.837µs I0327 15:21:58.353293 3959 namespace.go:142] [release-service] updating namespace I0327 15:21:58.368036 3959 namespace.go:142] [release-service] updating namespace I0327 15:21:58.423769 3959 namespace.go:142] [konflux-ui] updating namespace I0327 15:21:58.483701 3959 namespace.go:98] [konflux-info] adding namespace I0327 15:21:58.484479 3959 namespace.go:102] [konflux-info] adding namespace took 756.004µs I0327 15:21:58.521633 3959 namespace.go:142] [konflux-info] updating namespace I0327 15:21:58.551189 3959 namespace.go:98] [namespace-lister] adding namespace I0327 15:21:58.552031 3959 namespace.go:102] [namespace-lister] adding namespace took 825.779µs I0327 15:21:58.557121 3959 namespace.go:142] [konflux-info] updating namespace I0327 15:21:58.584765 3959 namespace.go:142] [namespace-lister] updating namespace I0327 15:21:58.589030 3959 namespace.go:98] [enterprise-contract-service] adding namespace I0327 15:21:58.589856 3959 namespace.go:102] [enterprise-contract-service] adding namespace took 801.616µs I0327 15:21:58.618177 3959 namespace.go:142] [namespace-lister] updating namespace I0327 15:21:58.620525 3959 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:21:58.624681 3959 namespace.go:142] [image-controller] updating namespace I0327 15:21:58.680320 3959 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:21:58.855142 3959 namespace.go:98] [default-tenant] adding namespace I0327 15:21:58.856161 3959 namespace.go:102] [default-tenant] adding namespace took 998.863µs I0327 15:21:58.857077 3959 namespace.go:142] [build-service] updating namespace I0327 15:21:58.926761 3959 namespace.go:98] [segment-bridge] adding namespace I0327 15:21:58.927642 3959 namespace.go:102] [segment-bridge] adding namespace took 861.306µs I0327 15:21:58.930182 3959 namespace.go:142] [default-tenant] updating namespace I0327 15:21:58.988189 3959 namespace.go:142] [default-tenant] updating namespace I0327 15:21:59.009199 3959 namespace.go:142] [segment-bridge] updating namespace I0327 15:21:59.045186 3959 namespace.go:142] [image-controller] updating namespace I0327 15:21:59.215118 3959 namespace.go:142] [segment-bridge] updating namespace I0327 15:21:59.255133 3959 namespace.go:142] [integration-service] updating namespace I0327 15:21:59.453098 3959 namespace.go:142] [release-service] updating namespace I0327 15:21:59.575077 3959 namespace.go:142] [build-service] updating namespace I0327 15:21:59.585827 3959 namespace.go:142] [image-controller] updating namespace I0327 15:21:59.596655 3959 namespace.go:142] [integration-service] updating namespace I0327 15:21:59.607136 3959 namespace.go:142] [release-service] updating namespace I0327 15:21:59.644054 3959 namespace.go:142] [build-service] updating namespace I0327 15:21:59.701074 3959 namespace.go:142] [image-controller] updating namespace I0327 15:21:59.799086 3959 namespace.go:142] [integration-service] updating namespace I0327 15:21:59.906437 3959 namespace.go:142] [release-service] updating namespace I0327 15:22:00.409308 3959 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 15:22:00.409933 3959 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 15:22:00.409955 3959 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 15:22:00.412388 3959 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel W0327 15:22:00.563288 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0327 15:22:00.568943 3959 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 15:22:00.568969 3959 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 15:22:00.569846 3959 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 15:22:01.090233 3959 namespace.go:142] [default-tenant] updating namespace I0327 15:22:01.308336 3959 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:22:01.521775 3959 namespace.go:142] [konflux-info] updating namespace I0327 15:22:01.672603 3959 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-2bgd6] creating logical port build-service_build-service-controller-manager-7df75dd4df-2bgd6 for pod on switch ip-10-0-1-144.ec2.internal I0327 15:22:01.672703 3959 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-2bgd6 I0327 15:22:01.689747 3959 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-2bgd6] pod update took 17.067892ms I0327 15:22:01.689767 3959 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-2bgd6] addLogicalPort annotation time took 17.107575ms I0327 15:22:01.690953 3959 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-2bgd6] addLogicalPort took 18.365688ms, libovsdb time 812.558µs I0327 15:22:01.709022 3959 namespace.go:142] [namespace-lister] updating namespace I0327 15:22:01.905455 3959 namespace.go:142] [segment-bridge] updating namespace I0327 15:22:02.031819 3959 namespace.go:142] [default-tenant] updating namespace I0327 15:22:02.034211 3959 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-2bgd6 67be5878e5d61a82195be70b6a00728d029333bd98211da1631028fd1b647d79 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-2bgd6 67be5878e5d61a82195be70b6a00728d029333bd98211da1631028fd1b647d79 network default NAD default] I0327 15:22:02.043096 3959 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-2bgd6, hostIfaceName: 67be5878e5d61a8, network: default, NAD default, SandboxID: "67be5878e5d61a82195be70b6a00728d029333bd98211da1631028fd1b647d79", PCI device ID: , UID: "2370f9ce-5325-4ea0-8555-550cff4abd1a", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0327 15:22:02.055039 3959 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:22:02.077449 3959 namespace.go:142] [konflux-info] updating namespace I0327 15:22:02.089778 3959 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-2bgd6 67be5878e5d61a82195be70b6a00728d029333bd98211da1631028fd1b647d79 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-2bgd6 67be5878e5d61a82195be70b6a00728d029333bd98211da1631028fd1b647d79 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:86:7e:16:97:08\",\"name\":\"67be5878e5d61a8\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18084713-8d8c-4fe1-8df6-bf700d8a1949\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:22:02.097242 3959 namespace.go:142] [namespace-lister] updating namespace I0327 15:22:02.112190 3959 namespace.go:142] [segment-bridge] updating namespace I0327 15:22:02.160872 3959 base_network_controller_pods.go:487] [default/konflux-ui/proxy-75d77ddb49-kvtfl] creating logical port konflux-ui_proxy-75d77ddb49-kvtfl for pod on switch ip-10-0-1-144.ec2.internal I0327 15:22:02.160992 3959 kube.go:256] Updating pod konflux-ui/proxy-75d77ddb49-kvtfl I0327 15:22:02.180772 3959 pod.go:62] [konflux-ui/proxy-75d77ddb49-kvtfl] pod update took 19.824912ms I0327 15:22:02.180793 3959 base_network_controller_pods.go:937] [default/konflux-ui/proxy-75d77ddb49-kvtfl] addLogicalPort annotation time took 19.852335ms I0327 15:22:02.182058 3959 pods.go:271] [konflux-ui/proxy-75d77ddb49-kvtfl] addLogicalPort took 21.205609ms, libovsdb time 906.462µs I0327 15:22:02.206017 3959 namespace.go:142] [default-tenant] updating namespace I0327 15:22:02.274498 3959 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:22:02.346917 3959 namespace.go:142] [konflux-info] updating namespace I0327 15:22:02.408512 3959 namespace.go:142] [namespace-lister] updating namespace I0327 15:22:02.466886 3959 namespace.go:142] [segment-bridge] updating namespace I0327 15:22:02.659228 3959 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-67bc7767bf-cfvbd] creating logical port integration-service_integration-service-controller-manager-67bc7767bf-cfvbd for pod on switch ip-10-0-1-144.ec2.internal I0327 15:22:02.659342 3959 kube.go:256] Updating pod integration-service/integration-service-controller-manager-67bc7767bf-cfvbd I0327 15:22:02.687602 3959 pod.go:62] [integration-service/integration-service-controller-manager-67bc7767bf-cfvbd] pod update took 28.292331ms I0327 15:22:02.687630 3959 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-67bc7767bf-cfvbd] addLogicalPort annotation time took 28.325685ms I0327 15:22:02.689418 3959 pods.go:271] [integration-service/integration-service-controller-manager-67bc7767bf-cfvbd] addLogicalPort took 30.208335ms, libovsdb time 1.311733ms I0327 15:22:02.875086 3959 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:02.875898 3959 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:03.904261 3959 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:03.904288 3959 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:04.043335 3959 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-1-144.ec2.internal I0327 15:22:04.043430 3959 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0327 15:22:04.055127 3959 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 11.721941ms I0327 15:22:04.055152 3959 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 11.751026ms I0327 15:22:04.056512 3959 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 13.195952ms, libovsdb time 961.003µs I0327 15:22:05.050588 3959 cni.go:403] [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default] I0327 15:22:05.059797 3959 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: 3f1c421963d8b2c, network: default, NAD default, SandboxID: "3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093", PCI device ID: , UID: "81d6488c-f09a-4746-99fc-616f64902728", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0327 15:22:05.108026 3959 cni.go:424] [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:ea:73:eb:04:bf\",\"name\":\"3f1c421963d8b2c\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/468152a4-2683-4a0b-84b3-fcc767dc5c35\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:22:05.246815 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0327 15:22:06.634955 3959 cni.go:403] [integration-service/integration-service-controller-manager-67bc7767bf-cfvbd 3cfe5185473ac63b63c3ddd99abe20150ce9fff2c9a684c88c04ad6e50db4c01 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-67bc7767bf-cfvbd 3cfe5185473ac63b63c3ddd99abe20150ce9fff2c9a684c88c04ad6e50db4c01 network default NAD default] I0327 15:22:06.643824 3959 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-67bc7767bf-cfvbd, hostIfaceName: 3cfe5185473ac63, network: default, NAD default, SandboxID: "3cfe5185473ac63b63c3ddd99abe20150ce9fff2c9a684c88c04ad6e50db4c01", PCI device ID: , UID: "5ac6ef14-5f97-4c46-a519-8025f93a7c06", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0327 15:22:06.689353 3959 cni.go:424] [integration-service/integration-service-controller-manager-67bc7767bf-cfvbd 3cfe5185473ac63b63c3ddd99abe20150ce9fff2c9a684c88c04ad6e50db4c01 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-67bc7767bf-cfvbd 3cfe5185473ac63b63c3ddd99abe20150ce9fff2c9a684c88c04ad6e50db4c01 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:fa:64:e9:0f:4a\",\"name\":\"3cfe5185473ac63\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3816b0d4-6c58-48c3-adbf-de00f5b3f03c\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:22:06.743474 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0327 15:22:06.765770 3959 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 15:22:10.777878 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:10.777972 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:11.225512 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf W0327 15:22:12.797345 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 15:22:12.797390 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 15:22:12.819189 3959 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0327 15:22:13.440164 3959 cni.go:403] [konflux-ui/proxy-75d77ddb49-kvtfl d6e2b1488b34506f3ef7c5afbb4ff7823128a6647cb03d07fe192c7006a5ee6d network default NAD default] ADD starting CNI request [konflux-ui/proxy-75d77ddb49-kvtfl d6e2b1488b34506f3ef7c5afbb4ff7823128a6647cb03d07fe192c7006a5ee6d network default NAD default] I0327 15:22:13.448702 3959 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-75d77ddb49-kvtfl, hostIfaceName: d6e2b1488b34506, network: default, NAD default, SandboxID: "d6e2b1488b34506f3ef7c5afbb4ff7823128a6647cb03d07fe192c7006a5ee6d", PCI device ID: , UID: "e7c42b64-38ba-458d-803c-0618e17d6a2e", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0327 15:22:13.499105 3959 cni.go:424] [konflux-ui/proxy-75d77ddb49-kvtfl d6e2b1488b34506f3ef7c5afbb4ff7823128a6647cb03d07fe192c7006a5ee6d network default NAD default] ADD finished CNI request [konflux-ui/proxy-75d77ddb49-kvtfl d6e2b1488b34506f3ef7c5afbb4ff7823128a6647cb03d07fe192c7006a5ee6d network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:bd:bd:ce:e9:ca\",\"name\":\"d6e2b1488b34506\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1e80419d-c34a-439a-bbe2-6ca0216a811e\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:22:14.313817 3959 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:14.313988 3959 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:16.256880 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 15:22:16.888290 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:16.888309 3959 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:18.343735 3959 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:18.343755 3959 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:21.790022 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:21.790044 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:21.790711 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:21.790730 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:25.322985 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:25.324778 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0327 15:22:25.637244 3959 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.637852 3959 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.886250 3959 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.886276 3959 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.892400 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-d4bhv I0327 15:22:25.892424 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-d4bhv I0327 15:22:49.638806 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 W0327 15:22:54.901501 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:54.901521 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:54.901535 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:54.901543 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:58.418146 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:58.418169 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 15:23:28.351821 3959 obj_retry.go:464] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:28.352469 3959 obj_retry.go:464] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:29.273496 3959 obj_retry.go:458] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:29.273518 3959 obj_retry.go:458] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:29.279214 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5f5844ff9b-p58tn I0327 15:23:29.279242 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5f5844ff9b-p58tn I0327 15:23:29.656746 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 15:23:35.765967 3959 cni.go:403] [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default] I0327 15:23:35.835065 3959 cni.go:424] [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux 3f1c421963d8b2ca5cccccdaa281f91a5dbc745f1143e342751644e084a09093 network default NAD default], result "{}", err I0327 15:23:35.886425 3959 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:35.886459 3959 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0327 15:23:35.888133 3959 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.52 I0327 15:23:35.888162 3959 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:35.922063 3959 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:35.922081 3959 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:35.924486 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 15:23:35.924505 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 15:24:03.644735 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0327 15:24:05.668683 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0327 15:24:06.858364 3959 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:06.858965 3959 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:07.491265 3959 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:07.491287 3959 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:07.498693 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd I0327 15:24:07.498719 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd I0327 15:24:08.562615 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=649 I0327 15:24:20.852482 3959 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:20.853117 3959 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:21.584325 3959 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:21.584350 3959 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:21.591803 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx I0327 15:24:21.591822 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx I0327 15:24:39.557508 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=45 I0327 15:24:42.557182 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=104 I0327 15:24:52.557772 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=72 I0327 15:25:45.675117 3959 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0327 15:25:46.661810 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0327 15:26:00.645417 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0327 15:26:43.558763 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=70 I0327 15:26:51.554124 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0327 15:27:14.681622 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0327 15:27:21.683677 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 15:29:13.684287 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 15:29:34.565615 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=13 I0327 15:30:04.645587 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:04.646198 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:05.523844 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:05.523891 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:12.659599 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0327 15:30:41.560781 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=26 I0327 15:31:36.059546 3959 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:36.059578 3959 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:36.064121 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-rmnrb I0327 15:31:36.064155 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-rmnrb I0327 15:31:42.028719 3959 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:42.028762 3959 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:42.031459 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-fwvnf I0327 15:31:42.031492 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-fwvnf I0327 15:31:45.041354 3959 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:45.041399 3959 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:45.045830 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-lnpnk I0327 15:31:45.045854 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-lnpnk I0327 15:31:57.559436 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0327 15:32:15.677882 3959 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:32:24.647854 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 15:32:34.641293 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0327 15:33:20.559997 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0327 15:33:21.956075 3959 namespace.go:98] [default-tenant-managed] adding namespace I0327 15:33:21.956971 3959 namespace.go:102] [default-tenant-managed] adding namespace took 872.462µs I0327 15:33:21.998772 3959 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:22.016153 3959 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:22.963579 3959 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:23.093348 3959 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:23.194238 3959 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:39.647428 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0327 15:33:56.683965 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 15:34:04.670722 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 I0327 15:34:11.730032 3959 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-krhcj for pod on switch ip-10-0-1-144.ec2.internal I0327 15:34:11.730189 3959 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj I0327 15:34:11.758000 3959 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj] pod update took 27.856413ms I0327 15:34:11.758022 3959 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj] addLogicalPort annotation time took 27.882329ms I0327 15:34:11.759251 3959 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj] addLogicalPort took 29.240151ms, libovsdb time 865.559µs I0327 15:34:12.091314 3959 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj b76144041098f71108cd2386362e0e3999d9a81fb61a4c146d04be7213402ab8 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj b76144041098f71108cd2386362e0e3999d9a81fb61a4c146d04be7213402ab8 network default NAD default] I0327 15:34:12.099550 3959 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-krhcj, hostIfaceName: b76144041098f71, network: default, NAD default, SandboxID: "b76144041098f71108cd2386362e0e3999d9a81fb61a4c146d04be7213402ab8", PCI device ID: , UID: "8ee6a138-af30-4c89-b675-5eb06f5a564c", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0327 15:34:12.147865 3959 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj b76144041098f71108cd2386362e0e3999d9a81fb61a4c146d04be7213402ab8 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-krhcj b76144041098f71108cd2386362e0e3999d9a81fb61a4c146d04be7213402ab8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:ed:62:4b:74:66\",\"name\":\"b76144041098f71\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33134a61-1e64-49ec-9ea7-290b8f588ba7\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:34:19.686173 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0327 15:34:26.561518 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 W0327 15:34:26.972031 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:34:26.972050 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:34:26.972056 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:34:26.972061 3959 util.go:959] Failed to build node endpoints for node ip-10-0-1-144.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0327 15:34:29.131579 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:29.132218 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:36.568551 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=45 I0327 15:34:39.557113 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0327 15:34:57.322796 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:57.323393 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:35:11.664517 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 15:35:20.541295 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:35:20.541876 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:36:23.563474 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=31 I0327 15:37:33.561762 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0327 15:37:37.011182 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-build-image-index-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:37:37.011333 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod I0327 15:37:37.038028 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod] pod update took 26.731928ms I0327 15:37:37.038051 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod] addLogicalPort annotation time took 26.758809ms I0327 15:37:37.039276 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod] addLogicalPort took 28.115162ms, libovsdb time 856.185µs I0327 15:37:37.368886 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default] I0327 15:37:37.377571 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-build-image-index-pod, hostIfaceName: bcf10f00844eb1b, network: default, NAD default, SandboxID: "bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e", PCI device ID: , UID: "59b87116-495d-4dd0-a247-48b5b5c7b3e4", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0327 15:37:37.425370 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:94:06:43:7d:13\",\"name\":\"bcf10f00844eb1b\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffb36ea2-26a9-4266-b0d8-588bd1d2cccf\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:37:37.500092 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:37:37.500641 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:37:38.687342 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 15:38:03.687763 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0327 15:38:21.093384 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:38:21.093552 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod I0327 15:38:21.125231 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod] pod update took 31.718691ms I0327 15:38:21.125254 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod] addLogicalPort annotation time took 31.748757ms I0327 15:38:21.126614 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod] addLogicalPort took 33.25125ms, libovsdb time 907.929µs I0327 15:38:21.215706 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default] I0327 15:38:21.308957 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod bcf10f00844eb1b77cd03b23680161c6683d8f71c1ef762686a8a31d9a98d85e network default NAD default], result "{}", err I0327 15:38:21.415490 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:21.415528 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod I0327 15:38:21.417234 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod, ips: 10.128.0.54 I0327 15:38:21.417263 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:21.522050 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default] I0327 15:38:21.530740 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod, hostIfaceName: 086c4cb04135722, network: default, NAD default, SandboxID: "086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b", PCI device ID: , UID: "5fca11c4-07c8-4824-a751-ac491363ac70", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0327 15:38:21.581883 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:64:86:27:ad:e9\",\"name\":\"086c4cb04135722\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7095f25-a11d-4f5c-9da9-66282f1378a5\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:38:22.207439 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:38:22.207583 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod I0327 15:38:22.225625 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod] pod update took 18.093028ms I0327 15:38:22.225652 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod] addLogicalPort annotation time took 18.122694ms I0327 15:38:22.226940 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod] addLogicalPort took 19.512523ms, libovsdb time 870.783µs I0327 15:38:22.641226 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default] I0327 15:38:22.649408 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod, hostIfaceName: a92e5cd36f8cdbf, network: default, NAD default, SandboxID: "a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1", PCI device ID: , UID: "2276af2f-ed85-482d-888e-c0882812b9f2", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0327 15:38:22.903496 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:12:92:4e:6d:11\",\"name\":\"a92e5cd36f8cdbf\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1619cbdd-b163-44f1-9831-2b20784f89bf\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:38:22.912034 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:38:22.912149 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod I0327 15:38:22.938849 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod] pod update took 26.733763ms I0327 15:38:22.938869 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod] addLogicalPort annotation time took 26.757522ms I0327 15:38:22.940289 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod] addLogicalPort took 28.27065ms, libovsdb time 1.05167ms I0327 15:38:22.948426 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:38:22.948533 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod I0327 15:38:22.969713 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod] pod update took 21.208455ms I0327 15:38:22.969733 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod] addLogicalPort annotation time took 21.232056ms I0327 15:38:22.970943 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod] addLogicalPort took 22.531465ms, libovsdb time 814.313µs I0327 15:38:23.323872 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default] I0327 15:38:23.333019 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod, hostIfaceName: 9ea936fd7c4804c, network: default, NAD default, SandboxID: "9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5", PCI device ID: , UID: "4d91219b-d20b-4c91-977f-d51bfa4281e1", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0327 15:38:23.383700 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:94:fd:45:57:ad\",\"name\":\"9ea936fd7c4804c\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8d510c1-753b-4c45-a6df-04c36d25f683\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:38:23.389468 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default] I0327 15:38:23.399192 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod, hostIfaceName: 389945ac401969b, network: default, NAD default, SandboxID: "389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9", PCI device ID: , UID: "e9ae4ed5-1f93-463f-8e8f-5d9a3075150b", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0327 15:38:23.446771 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:87:7f:e2:ff:1f\",\"name\":\"389945ac401969b\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05dcbf8c-cc00-4c78-b59e-76608edff0cb\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:38:24.621393 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:38:24.621610 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod I0327 15:38:24.638767 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod] pod update took 17.19831ms I0327 15:38:24.638794 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod] addLogicalPort annotation time took 17.229721ms I0327 15:38:24.640367 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod] addLogicalPort took 18.995255ms, libovsdb time 1.027343ms I0327 15:38:24.837389 3959 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:38:24.837509 3959 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod I0327 15:38:24.854168 3959 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod] pod update took 16.694024ms I0327 15:38:24.854198 3959 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod] addLogicalPort annotation time took 16.72596ms I0327 15:38:24.855846 3959 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod] addLogicalPort took 18.47868ms, libovsdb time 1.041439ms I0327 15:38:25.040272 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default] I0327 15:38:25.050064 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod, hostIfaceName: 1e89664f450209d, network: default, NAD default, SandboxID: "1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979", PCI device ID: , UID: "3bf7f62d-5dcd-47f2-85af-b36c2d8b903b", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0327 15:38:25.106924 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:34:77:4c:a5:ef\",\"name\":\"1e89664f450209d\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/536c55a8-de09-42b9-bda4-155bde98419b\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:38:25.251008 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default] I0327 15:38:25.261898 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod, hostIfaceName: d0c1b4dc3e9ba1c, network: default, NAD default, SandboxID: "d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83", PCI device ID: , UID: "3a0e69fb-71e2-4056-9a8f-8a58ab36a44f", MAC: 0a:58:0a:80:00:3c, IPs: [10.128.0.60/23] I0327 15:38:25.322999 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:ab:65:94:c7:fb\",\"name\":\"d0c1b4dc3e9ba1c\"},{\"mac\":\"0a:58:0a:80:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3c3c1725-5592-4779-a5f2-5b47da159b62\"}],\"ips\":[{\"address\":\"10.128.0.60/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:38:42.809940 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:42.810535 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:57.650547 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 15:39:03.616130 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default] I0327 15:39:03.718547 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod a92e5cd36f8cdbf3842bee21e86b3d4d8ab22cc9bfa595440af763a4de8a0cf1 network default NAD default], result "{}", err I0327 15:39:03.818536 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:03.818578 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod I0327 15:39:03.820363 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod, ips: 10.128.0.56 I0327 15:39:03.820400 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:04.622779 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default] I0327 15:39:04.727522 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod 9ea936fd7c4804cb5794f71fff5dda20f37d78f9bbfeb6908a75c224956424e5 network default NAD default], result "{}", err I0327 15:39:04.957932 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:04.958054 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod I0327 15:39:04.959904 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod, ips: 10.128.0.57 I0327 15:39:04.959942 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:08.673671 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0327 15:39:21.284701 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:21.285382 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.009267 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.009847 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.282943 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.283727 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:34.836306 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default] I0327 15:39:34.930488 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod 389945ac401969b1c516d5b6e7df6758fe570896e7e105ab5593af00ef747bf9 network default NAD default], result "{}", err I0327 15:39:35.054176 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:35.054217 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod I0327 15:39:35.056025 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod, ips: 10.128.0.58 I0327 15:39:35.056109 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:50.955112 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default] I0327 15:39:51.034297 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod d0c1b4dc3e9ba1ce47def6fe3b27379cd8c20f71a7665269c33cf88d3279dd83 network default NAD default], result "{}", err I0327 15:39:51.121477 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:51.121516 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod I0327 15:39:51.123046 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod, ips: 10.128.0.60 I0327 15:39:51.123078 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:51.962595 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default] I0327 15:39:52.053620 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod 1e89664f450209d90374a39aa53f96f6ee5672594f45b3520ffe2c85dcc5d979 network default NAD default], result "{}", err I0327 15:39:52.173219 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:52.173255 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod I0327 15:39:52.174743 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod, ips: 10.128.0.59 I0327 15:39:52.174777 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:00.026985 3959 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default] I0327 15:40:00.116827 3959 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod 086c4cb04135722ade001f91311a858753d275ab38f4fcde363eeeeeca5bb59b network default NAD default], result "{}", err I0327 15:40:00.285906 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:00.285935 3959 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod I0327 15:40:00.287525 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod, ips: 10.128.0.55 I0327 15:40:00.287560 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:04.571712 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:04.572274 3959 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:05.407225 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.407247 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.445995 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.446013 3959 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.450012 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 I0327 15:40:05.450036 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 I0327 15:40:16.680141 3959 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0327 15:40:44.560020 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0327 15:40:53.649734 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0327 15:41:27.644474 3959 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0327 15:41:42.244684 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:41:42.245330 3959 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:41:45.891622 3959 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-fxdi-x94ws-verify-pod] creating logical port default-tenant_my-integration-test-fxdi-x94ws-verify-pod for pod on switch ip-10-0-1-144.ec2.internal I0327 15:41:45.891805 3959 kube.go:256] Updating pod default-tenant/my-integration-test-fxdi-x94ws-verify-pod I0327 15:41:45.922818 3959 pod.go:62] [default-tenant/my-integration-test-fxdi-x94ws-verify-pod] pod update took 31.057267ms I0327 15:41:45.922842 3959 base_network_controller_pods.go:937] [default/default-tenant/my-integration-test-fxdi-x94ws-verify-pod] addLogicalPort annotation time took 31.085249ms I0327 15:41:45.924065 3959 pods.go:271] [default-tenant/my-integration-test-fxdi-x94ws-verify-pod] addLogicalPort took 32.457879ms, libovsdb time 830.916µs I0327 15:41:46.258637 3959 cni.go:403] [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default] ADD starting CNI request [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default] I0327 15:41:46.267059 3959 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-fxdi-x94ws-verify-pod, hostIfaceName: dd6c5f6a32f63b1, network: default, NAD default, SandboxID: "dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec", PCI device ID: , UID: "ececf827-d5aa-41be-a95d-fe19a9dc8d12", MAC: 0a:58:0a:80:00:3d, IPs: [10.128.0.61/23] I0327 15:41:46.314499 3959 cni.go:424] [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default] ADD finished CNI request [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:e3:d8:1d:cf:7d\",\"name\":\"dd6c5f6a32f63b1\"},{\"mac\":\"0a:58:0a:80:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/acb76b09-f203-48f4-a6cd-5d5ee7684e31\"}],\"ips\":[{\"address\":\"10.128.0.61/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:42:09.861563 3959 cni.go:403] [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default] DEL starting CNI request [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default] I0327 15:42:09.947402 3959 cni.go:424] [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default] DEL finished CNI request [default-tenant/my-integration-test-fxdi-x94ws-verify-pod dd6c5f6a32f63b12039a7db778b2353c2f45226a4fc943a1e15a76ebe33369ec network default NAD default], result "{}", err I0327 15:42:10.064457 3959 obj_retry.go:464] Detected object default-tenant/my-integration-test-fxdi-x94ws-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:10.064491 3959 pods.go:176] Deleting pod: default-tenant/my-integration-test-fxdi-x94ws-verify-pod I0327 15:42:10.065973 3959 pods.go:236] Attempting to release IPs for pod: default-tenant/my-integration-test-fxdi-x94ws-verify-pod, ips: 10.128.0.61 I0327 15:42:10.066001 3959 obj_retry.go:464] Detected object default-tenant/my-integration-test-fxdi-x94ws-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:25.562363 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=11 I0327 15:42:40.712745 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:40.713297 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:01.686332 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0327 15:43:06.667330 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0327 15:43:06.915951 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:06.916526 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:07.688611 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0327 15:43:14.694059 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 I0327 15:43:14.946290 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:14.946853 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:17.984182 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:17.984754 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:32.115749 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:32.116259 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:42.195783 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:42.196329 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:07.413006 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:07.413549 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:14.571267 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=313 I0327 15:44:16.827411 3959 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:16.827989 3959 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:17.321278 3959 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:44:17.321302 3959 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:44:17.324507 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-0426a73a44-0 I0327 15:44:17.324531 3959 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-0426a73a44-0 I0327 15:44:17.487737 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:17.488310 3959 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:21.563893 3959 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0327 15:44:32.676963 3959 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6