+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-37.ec2.internal ++ [[ -n ip-10-0-1-37.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-37.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0329 02:51:49.421048169 - disable conntrack on geneve port' I0329 02:51:49.421048169 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0329 02:51:49.438707448 - starting ovnkube-node + echo 'I0329 02:51:49.438707448 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-37.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-37.ec2.internal --init-node ip-10-0-1-37.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-37.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0329 02:51:49.479312 3827 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0329 02:51:49.479357 3827 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-457e589fce.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0329 02:51:49.480725 3827 kube.go:419] Waiting for certificate I0329 02:51:49.480797 3827 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0329 02:51:49.480872 3827 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0329 02:51:49.481715 3827 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0329 02:51:49.508077 3827 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:49.508097 3827 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:49.510384 3827 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:49.513308 3827 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-28knf" I0329 02:51:49.523194 3827 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-28knf" I0329 02:51:49.523264 3827 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:50.480804 3827 kube.go:426] Certificate found I0329 02:51:50.481383 3827 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0329 02:51:50.482265 3827 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0329 02:51:50.527082 3827 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-30 02:46:49 +0000 UTC" deadline="2026-03-29 21:23:10.010691498 +0000 UTC" I0329 02:51:50.527133 3827 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h31m19.483568379s" I0329 02:51:50.545156 3827 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0329 02:51:50.564585 3827 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0329 02:51:50.565631 3827 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0329 02:51:50.565706 3827 factory.go:531] Starting watch factory I0329 02:51:50.565787 3827 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565807 3827 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565839 3827 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565861 3827 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565888 3827 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565906 3827 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565887 3827 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.565955 3827 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.566069 3827 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.566129 3827 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.566446 3827 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.566526 3827 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.572226 3827 controller_manager.go:368] Starting the ovnkube controller I0329 02:51:50.572237 3827 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-37.ec2.internal I0329 02:51:50.572285 3827 controller_manager.go:393] NBDB zone sync took: 40.753µs I0329 02:51:50.572293 3827 factory.go:531] Starting watch factory I0329 02:51:50.587980 3827 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.587980 3827 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.588218 3827 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0329 02:51:50.590115 3827 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.590247 3827 ovn_northd.go:121] Registering OVN northd metrics I0329 02:51:50.591309 3827 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.592771 3827 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.593072 3827 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0329 02:51:50.593124 3827 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0329 02:51:50.601414 3827 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:50.609733 3827 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0329 02:51:50.671856 3827 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0329 02:51:50.671873 3827 factory.go:1890] *v1.Pod informer cache synced successfully I0329 02:51:50.671877 3827 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0329 02:51:50.671880 3827 factory.go:1890] *v1.Namespace informer cache synced successfully I0329 02:51:50.671883 3827 factory.go:1890] *v1.Node informer cache synced successfully I0329 02:51:50.671886 3827 factory.go:1890] *v1.Service informer cache synced successfully I0329 02:51:50.671993 3827 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:50.672020 3827 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:50.672052 3827 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:50.672107 3827 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:50.675098 3827 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:50.675096 3827 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:50.677365 3827 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0329 02:51:50.677375 3827 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0329 02:51:50.677435 3827 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.677447 3827 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.677530 3827 factory.go:1890] *v1.Namespace informer cache synced successfully I0329 02:51:50.677541 3827 factory.go:1890] *v1.Node informer cache synced successfully I0329 02:51:50.677544 3827 factory.go:1890] *v1.Service informer cache synced successfully I0329 02:51:50.677548 3827 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0329 02:51:50.677551 3827 factory.go:1890] *v1.Pod informer cache synced successfully I0329 02:51:50.677554 3827 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0329 02:51:50.680002 3827 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.683165 3827 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0329 02:51:50.683177 3827 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0329 02:51:50.683240 3827 factory.go:1890] *v1.EgressIP informer cache synced successfully I0329 02:51:50.683310 3827 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.683321 3827 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.685945 3827 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.689006 3827 factory.go:1890] *v1.EgressIP informer cache synced successfully I0329 02:51:50.689086 3827 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0329 02:51:50.689140 3827 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.689152 3827 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.692109 3827 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.694326 3827 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0329 02:51:50.694387 3827 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.694398 3827 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.694474 3827 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0329 02:51:50.696939 3827 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.700021 3827 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0329 02:51:50.700095 3827 factory.go:1890] *v1.EgressService informer cache synced successfully I0329 02:51:50.700147 3827 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.700159 3827 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.702938 3827 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.705986 3827 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0329 02:51:50.706000 3827 factory.go:1890] *v1.EgressService informer cache synced successfully I0329 02:51:50.706415 3827 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0329 02:51:50.706448 3827 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0329 02:51:50.709075 3827 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0329 02:51:50.711079 3827 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0329 02:51:50.711098 3827 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0329 02:51:50.711155 3827 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.711169 3827 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.711189 3827 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.711206 3827 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.713933 3827 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.714111 3827 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:50.716123 3827 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0329 02:51:50.716134 3827 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0329 02:51:50.716141 3827 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0329 02:51:50.716147 3827 factory.go:636] Watch Factory start up complete, took: 150.445798ms I0329 02:51:50.716222 3827 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0329 02:51:50.721958 3827 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0329 02:51:50.721969 3827 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0329 02:51:50.721976 3827 factory.go:636] Watch Factory start up complete, took: 149.681264ms I0329 02:51:50.721985 3827 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-37.ec2.internal" zone I0329 02:51:50.748790 3827 default_node_network_controller.go:709] Initializing the default node network controller I0329 02:51:50.800113 3827 udn_isolation.go:101] Starting UDN host isolation manager I0329 02:51:50.807592 3827 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0329 02:51:50.871719 3827 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0329 02:51:50.871814 3827 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0329 02:51:50.871830 3827 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0329 02:51:50.906455 3827 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0329 02:51:50.906518 3827 default_node_network_controller.go:808] Node ip-10-0-1-37.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0329 02:51:50.912202 3827 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.37"] k8s.ovn.org/zone-name:ip-10-0-1-37.ec2.internal] on node ip-10-0-1-37.ec2.internal I0329 02:51:50.933929 3827 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0329 02:51:50.939416 3827 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0329 02:51:50.939587 3827 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0329 02:51:50.939760 3827 gateway_init.go:263] Preparing Gateway I0329 02:51:50.939776 3827 gateway_shared_intf.go:1658] Creating new gateway I0329 02:51:50.972423 3827 controller_manager.go:427] Waiting for node in zone sync took: 250.424452ms I0329 02:51:50.979896 3827 controller_manager.go:608] Switching to transit router for layer2 networks I0329 02:51:50.979910 3827 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-37.ec2.internal I0329 02:51:50.997383 3827 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0329 02:51:51.003543 3827 controller_manager.go:288] SCTP support detected in OVN I0329 02:51:51.004114 3827 services_controller.go:69] Creating services controller for network=default I0329 02:51:51.004198 3827 udn_enabled_svc.go:80] Waiting for service informer to sync I0329 02:51:51.004277 3827 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0329 02:51:51.004314 3827 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0329 02:51:51.004331 3827 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0329 02:51:51.004354 3827 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0329 02:51:51.004497 3827 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0329 02:51:51.004558 3827 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0329 02:51:51.004574 3827 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 44.957µs I0329 02:51:51.004596 3827 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 28.586µs I0329 02:51:51.004607 3827 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 47.883µs I0329 02:51:51.004618 3827 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0329 02:51:51.004626 3827 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0329 02:51:51.004638 3827 default_network_controller.go:321] Starting the default network controller I0329 02:51:51.004644 3827 udn_enabled_svc.go:95] Performing full resync I0329 02:51:51.004768 3827 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 11.898µs I0329 02:51:51.004927 3827 udn_enabled_svc.go:99] Waiting for handler to sync I0329 02:51:51.004939 3827 udn_enabled_svc.go:104] Starting worker I0329 02:51:51.012980 3827 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0329 02:51:51.013061 3827 repair.go:33] Syncing exgw routes took 63.067µs I0329 02:51:51.013083 3827 default_network_controller.go:407] Starting all the Watchers... I0329 02:51:51.013209 3827 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0329 02:51:51.013209 3827 namespace.go:98] [openshift-authentication-operator] adding namespace I0329 02:51:51.013220 3827 namespace.go:98] [kube-node-lease] adding namespace I0329 02:51:51.013224 3827 namespace.go:98] [default] adding namespace I0329 02:51:51.013240 3827 namespace.go:98] [openshift-ingress] adding namespace I0329 02:51:51.013251 3827 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0329 02:51:51.013259 3827 namespace.go:98] [openshift-backplane-cse] adding namespace I0329 02:51:51.013268 3827 namespace.go:98] [openshift-controller-manager] adding namespace I0329 02:51:51.013274 3827 namespace.go:98] [openshift-operators] adding namespace I0329 02:51:51.013276 3827 namespace.go:98] [openshift-config-managed] adding namespace I0329 02:51:51.013283 3827 namespace.go:98] [openshift-machine-api] adding namespace I0329 02:51:51.013343 3827 namespace.go:98] [openshift-console] adding namespace I0329 02:51:51.013349 3827 namespace.go:98] [openshift-host-network] adding namespace I0329 02:51:51.013360 3827 namespace.go:98] [openshift-service-ca-operator] adding namespace I0329 02:51:51.013364 3827 namespace.go:98] [openshift-kube-scheduler] adding namespace I0329 02:51:51.014147 3827 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 927.972µs I0329 02:51:51.014164 3827 namespace.go:98] [kube-public] adding namespace I0329 02:51:51.014652 3827 namespace.go:102] [kube-public] adding namespace took 481.804µs I0329 02:51:51.014667 3827 namespace.go:98] [openshift-kube-apiserver] adding namespace I0329 02:51:51.015122 3827 namespace.go:102] [openshift-authentication-operator] adding namespace took 1.900128ms I0329 02:51:51.015133 3827 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0329 02:51:51.015254 3827 iptables.go:108] Creating table: filter chain: FORWARD I0329 02:51:51.015564 3827 namespace.go:102] [kube-node-lease] adding namespace took 2.333645ms I0329 02:51:51.015574 3827 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0329 02:51:51.015974 3827 namespace.go:102] [default] adding namespace took 2.739668ms I0329 02:51:51.015989 3827 namespace.go:98] [openshift-ingress-operator] adding namespace I0329 02:51:51.016480 3827 namespace.go:102] [openshift-ingress] adding namespace took 3.229148ms I0329 02:51:51.016490 3827 namespace.go:98] [openshift-multus] adding namespace I0329 02:51:51.016995 3827 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 3.737828ms I0329 02:51:51.017011 3827 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0329 02:51:51.017472 3827 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.207531ms I0329 02:51:51.017483 3827 namespace.go:98] [openshift-backplane-srep] adding namespace I0329 02:51:51.017873 3827 namespace.go:102] [openshift-controller-manager] adding namespace took 4.594636ms I0329 02:51:51.017891 3827 namespace.go:98] [openshift-node] adding namespace I0329 02:51:51.018298 3827 namespace.go:102] [openshift-operators] adding namespace took 5.01636ms I0329 02:51:51.018313 3827 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0329 02:51:51.018806 3827 namespace.go:102] [openshift-config-managed] adding namespace took 5.519023ms I0329 02:51:51.018820 3827 namespace.go:98] [openshift-must-gather-operator] adding namespace I0329 02:51:51.019337 3827 namespace.go:102] [openshift-machine-api] adding namespace took 6.044351ms I0329 02:51:51.019349 3827 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0329 02:51:51.019878 3827 namespace.go:102] [openshift-console] adding namespace took 6.521234ms I0329 02:51:51.019893 3827 namespace.go:98] [openshift-config-operator] adding namespace I0329 02:51:51.020443 3827 namespace.go:102] [openshift-host-network] adding namespace took 7.083436ms I0329 02:51:51.020460 3827 namespace.go:98] [openshift-network-diagnostics] adding namespace I0329 02:51:51.020947 3827 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.58141ms I0329 02:51:51.020960 3827 namespace.go:98] [openshift-console-user-settings] adding namespace I0329 02:51:51.021408 3827 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.03806ms I0329 02:51:51.021422 3827 namespace.go:98] [openshift-logging] adding namespace I0329 02:51:51.021782 3827 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.110159ms I0329 02:51:51.021793 3827 namespace.go:98] [openshift-backplane-csm] adding namespace I0329 02:51:51.021969 3827 iptables.go:108] Creating table: filter chain: OUTPUT I0329 02:51:51.022159 3827 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.02211ms I0329 02:51:51.022170 3827 namespace.go:98] [openshift-infra] adding namespace I0329 02:51:51.022614 3827 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.035182ms I0329 02:51:51.022629 3827 namespace.go:98] [openshift-customer-monitoring] adding namespace W0329 02:51:51.022974 3827 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-7hs79: pod openshift-multus/network-metrics-daemon-7hs79: no pod IPs found I0329 02:51:51.023017 3827 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.022885ms I0329 02:51:51.023045 3827 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0329 02:51:51.023438 3827 namespace.go:102] [openshift-multus] adding namespace took 6.941264ms I0329 02:51:51.023453 3827 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0329 02:51:51.023926 3827 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.910058ms I0329 02:51:51.023938 3827 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0329 02:51:51.024345 3827 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.856525ms I0329 02:51:51.024361 3827 namespace.go:98] [openshift-image-registry] adding namespace I0329 02:51:51.024683 3827 namespace.go:102] [openshift-node] adding namespace took 6.78327ms I0329 02:51:51.024700 3827 namespace.go:98] [openshift-ingress-canary] adding namespace I0329 02:51:51.025115 3827 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.796466ms I0329 02:51:51.025127 3827 namespace.go:98] [openshift-authentication] adding namespace I0329 02:51:51.025630 3827 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.802994ms I0329 02:51:51.025646 3827 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0329 02:51:51.025987 3827 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.631116ms I0329 02:51:51.026001 3827 namespace.go:98] [openshift-operators-redhat] adding namespace W0329 02:51:51.026486 3827 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-zz6r9: pod openshift-network-diagnostics/network-check-target-zz6r9: no pod IPs found I0329 02:51:51.026536 3827 namespace.go:102] [openshift-config-operator] adding namespace took 6.635939ms I0329 02:51:51.026553 3827 namespace.go:98] [openshift-etcd] adding namespace I0329 02:51:51.027011 3827 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.545962ms I0329 02:51:51.027026 3827 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0329 02:51:51.027478 3827 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.51129ms I0329 02:51:51.027493 3827 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0329 02:51:51.027928 3827 namespace.go:102] [openshift-logging] adding namespace took 6.501951ms I0329 02:51:51.027940 3827 namespace.go:98] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] adding namespace I0329 02:51:51.028341 3827 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.542437ms I0329 02:51:51.028358 3827 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0329 02:51:51.028713 3827 namespace.go:102] [openshift-infra] adding namespace took 6.536817ms I0329 02:51:51.028727 3827 namespace.go:98] [openshift-backplane-cee] adding namespace I0329 02:51:51.028928 3827 gateway_shared_intf.go:1787] Gateway Creation Complete I0329 02:51:51.028942 3827 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-37.ec2.internal","mac-address":"0a:ff:e4:b6:c5:ef","ip-addresses":["10.0.1.37/24"],"ip-address":"10.0.1.37/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:94753d28-646c-4b21-9f3b-9a0207ae18b0 k8s.ovn.org/node-encap-ips:["10.0.1.37"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.37/24"} k8s.ovn.org/zone-name:ip-10-0-1-37.ec2.internal] on node ip-10-0-1-37.ec2.internal I0329 02:51:51.029129 3827 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.495118ms I0329 02:51:51.029145 3827 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0329 02:51:51.029607 3827 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.557097ms I0329 02:51:51.029619 3827 namespace.go:98] [openshift] adding namespace I0329 02:51:51.029963 3827 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.505011ms I0329 02:51:51.029974 3827 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0329 02:51:51.030387 3827 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.444303ms I0329 02:51:51.030398 3827 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0329 02:51:51.030859 3827 namespace.go:102] [openshift-image-registry] adding namespace took 6.491191ms I0329 02:51:51.030873 3827 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0329 02:51:51.031288 3827 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.582914ms I0329 02:51:51.031299 3827 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0329 02:51:51.031830 3827 namespace.go:102] [openshift-authentication] adding namespace took 6.698691ms I0329 02:51:51.031842 3827 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0329 02:51:51.032328 3827 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.677354ms I0329 02:51:51.032339 3827 namespace.go:98] [openshift-route-controller-manager] adding namespace I0329 02:51:51.032812 3827 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.805927ms I0329 02:51:51.032822 3827 namespace.go:98] [openshift-console-operator] adding namespace I0329 02:51:51.033309 3827 namespace.go:102] [openshift-etcd] adding namespace took 6.751079ms I0329 02:51:51.033328 3827 namespace.go:98] [openshift-backplane-tam] adding namespace I0329 02:51:51.033737 3827 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.690576ms I0329 02:51:51.033748 3827 namespace.go:98] [openshift-machine-config-operator] adding namespace I0329 02:51:51.034634 3827 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.134689ms I0329 02:51:51.034648 3827 namespace.go:98] [openshift-network-console] adding namespace I0329 02:51:51.035117 3827 namespace.go:102] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] adding namespace took 7.172336ms I0329 02:51:51.035130 3827 namespace.go:98] [openshift-dns-operator] adding namespace I0329 02:51:51.035574 3827 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.210549ms I0329 02:51:51.035586 3827 namespace.go:98] [openshift-backplane] adding namespace I0329 02:51:51.036006 3827 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.273701ms I0329 02:51:51.036019 3827 namespace.go:98] [openshift-cluster-version] adding namespace I0329 02:51:51.036502 3827 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.353664ms I0329 02:51:51.036511 3827 namespace.go:98] [openshift-monitoring] adding namespace I0329 02:51:51.036912 3827 namespace.go:102] [openshift] adding namespace took 7.287929ms I0329 02:51:51.036926 3827 namespace.go:98] [openshift-package-operator] adding namespace I0329 02:51:51.037414 3827 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.435822ms I0329 02:51:51.037424 3827 namespace.go:98] [kube-system] adding namespace I0329 02:51:51.037889 3827 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.485734ms I0329 02:51:51.037905 3827 namespace.go:98] [openshift-apiserver] adding namespace I0329 02:51:51.038301 3827 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.423309ms I0329 02:51:51.038314 3827 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0329 02:51:51.038728 3827 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.423267ms I0329 02:51:51.038741 3827 namespace.go:98] [openshift-backplane-mobb] adding namespace I0329 02:51:51.039177 3827 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.330665ms I0329 02:51:51.039190 3827 namespace.go:98] [openshift-network-node-identity] adding namespace I0329 02:51:51.039632 3827 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.288147ms I0329 02:51:51.040133 3827 namespace.go:102] [openshift-console-operator] adding namespace took 7.305656ms I0329 02:51:51.040148 3827 namespace.go:98] [dedicated-admin] adding namespace I0329 02:51:51.040519 3827 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.185261ms I0329 02:51:51.040531 3827 namespace.go:98] [openshift-config] adding namespace I0329 02:51:51.040963 3827 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.211846ms I0329 02:51:51.040973 3827 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0329 02:51:51.041442 3827 namespace.go:102] [openshift-network-console] adding namespace took 6.78951ms I0329 02:51:51.041454 3827 namespace.go:98] [openshift-network-operator] adding namespace I0329 02:51:51.041880 3827 namespace.go:102] [openshift-dns-operator] adding namespace took 6.745323ms I0329 02:51:51.041889 3827 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0329 02:51:51.042311 3827 namespace.go:102] [openshift-backplane] adding namespace took 6.720372ms I0329 02:51:51.042745 3827 namespace.go:102] [openshift-cluster-version] adding namespace took 6.720725ms I0329 02:51:51.042757 3827 namespace.go:98] [openshift-insights] adding namespace I0329 02:51:51.043294 3827 namespace.go:102] [openshift-monitoring] adding namespace took 6.777866ms I0329 02:51:51.043307 3827 namespace.go:98] [openshift-marketplace] adding namespace I0329 02:51:51.043703 3827 namespace.go:102] [openshift-package-operator] adding namespace took 6.772151ms I0329 02:51:51.044175 3827 namespace.go:102] [kube-system] adding namespace took 6.745464ms I0329 02:51:51.044620 3827 namespace.go:102] [openshift-apiserver] adding namespace took 6.710776ms I0329 02:51:51.045059 3827 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.737843ms I0329 02:51:51.045500 3827 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.754344ms I0329 02:51:51.045935 3827 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.740861ms I0329 02:51:51.046488 3827 namespace.go:102] [dedicated-admin] adding namespace took 6.3331ms I0329 02:51:51.046504 3827 namespace.go:98] [openshift-apiserver-operator] adding namespace I0329 02:51:51.046960 3827 namespace.go:102] [openshift-config] adding namespace took 6.423557ms I0329 02:51:51.046975 3827 namespace.go:98] [openshift-dns] adding namespace I0329 02:51:51.047436 3827 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.457223ms I0329 02:51:51.047908 3827 namespace.go:102] [openshift-network-operator] adding namespace took 6.448221ms I0329 02:51:51.048358 3827 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.461537ms I0329 02:51:51.048758 3827 namespace.go:102] [openshift-insights] adding namespace took 5.995607ms I0329 02:51:51.049209 3827 namespace.go:102] [openshift-marketplace] adding namespace took 5.89551ms I0329 02:51:51.049403 3827 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:51.049758 3827 namespace.go:102] [openshift-apiserver-operator] adding namespace took 3.248489ms I0329 02:51:51.050181 3827 namespace.go:102] [openshift-dns] adding namespace took 3.199393ms I0329 02:51:51.050585 3827 zone_ic_handler.go:186] Time taken to create transit switch: 279.414µs I0329 02:51:51.050654 3827 master.go:532] Adding or Updating local node "ip-10-0-1-37.ec2.internal" for network "default" I0329 02:51:51.051293 3827 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-173.ec2.internal for the network default I0329 02:51:51.052378 3827 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-173.ec2.internal" on network "default" took: 1.102368ms W0329 02:51:51.059051 3827 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-7hs79: pod openshift-multus/network-metrics-daemon-7hs79: no pod IPs found W0329 02:51:51.059078 3827 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-zz6r9: pod openshift-network-diagnostics/network-check-target-zz6r9: no pod IPs found I0329 02:51:51.060093 3827 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-37.ec2.internal for the network default I0329 02:51:51.060865 3827 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0329 02:51:51.060905 3827 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0329 02:51:51.060925 3827 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0329 02:51:51.060960 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-37.ec2.internal in network "default" I0329 02:51:51.061014 3827 node_tracker.go:169] Node ip-10-0-1-37.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:51.061027 3827 services_controller.go:574] Full service sync requested for network=default I0329 02:51:51.061058 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-173.ec2.internal in network "default" I0329 02:51:51.061092 3827 node_tracker.go:169] Node ip-10-0-2-173.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:51.061102 3827 services_controller.go:574] Full service sync requested for network=default I0329 02:51:51.161817 3827 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0329 02:51:51.161838 3827 services_controller.go:208] Setting up event handlers for services for network=default I0329 02:51:51.161882 3827 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0329 02:51:51.161914 3827 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0329 02:51:51.161925 3827 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0329 02:51:51.261979 3827 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0329 02:51:51.262000 3827 repair.go:60] Starting repairing loop for services I0329 02:51:51.262099 3827 repair.go:131] Deleted 0 stale service LBs I0329 02:51:51.262118 3827 repair.go:137] Deleted 0 stale Chassis Template Vars I0329 02:51:51.262134 3827 repair.go:62] Finished repairing loop for services: 136.204µs I0329 02:51:51.262157 3827 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0329 02:51:51.262167 3827 services_controller.go:254] Starting workers for network=default I0329 02:51:51.262546 3827 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-mhrm8 of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0329 02:51:51.262570 3827 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-7hs79] creating logical port openshift-multus_network-metrics-daemon-7hs79 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:51.262656 3827 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-zz6r9] creating logical port openshift-network-diagnostics_network-check-target-zz6r9 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:51.262704 3827 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-7hs79 I0329 02:51:51.262753 3827 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-zz6r9 W0329 02:51:51.266670 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0329 02:51:51.267361 3827 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0329 02:51:51.287392 3827 pod.go:62] [openshift-network-diagnostics/network-check-target-zz6r9] pod update took 24.667798ms I0329 02:51:51.287413 3827 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-zz6r9] addLogicalPort annotation time took 24.68933ms I0329 02:51:51.287540 3827 pod.go:62] [openshift-multus/network-metrics-daemon-7hs79] pod update took 24.928433ms I0329 02:51:51.287557 3827 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-7hs79] addLogicalPort annotation time took 24.947434ms I0329 02:51:51.288306 3827 pods.go:271] [openshift-network-diagnostics/network-check-target-zz6r9] addLogicalPort took 25.654207ms, libovsdb time 700.542µs I0329 02:51:51.288688 3827 pods.go:271] [openshift-multus/network-metrics-daemon-7hs79] addLogicalPort took 26.128635ms, libovsdb time 918.041µs I0329 02:51:51.289020 3827 admin_network_policy_controller.go:227] Starting controller default-network-controller I0329 02:51:51.289056 3827 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0329 02:51:51.289068 3827 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0329 02:51:51.289073 3827 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0329 02:51:51.289247 3827 repair.go:29] Repairing admin network policies took 169.5µs I0329 02:51:51.289342 3827 repair.go:92] Repairing baseline admin network policies took 87.021µs I0329 02:51:51.289698 3827 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0329 02:51:51.289723 3827 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0329 02:51:51.291967 3827 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0329 02:51:51.291992 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0329 02:51:51.292001 3827 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0329 02:51:51.293437 3827 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0329 02:51:51.293460 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0329 02:51:51.293473 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0329 02:51:51.295085 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0329 02:51:51.295103 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0329 02:51:51.295110 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0329 02:51:51.295424 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0329 02:51:51.295446 3827 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0329 02:51:51.295463 3827 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0329 02:51:51.297746 3827 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0329 02:51:51.297765 3827 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0329 02:51:51.297775 3827 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0329 02:51:51.298611 3827 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0329 02:51:51.298635 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0329 02:51:51.298647 3827 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0329 02:51:51.299308 3827 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0329 02:51:51.299325 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0329 02:51:51.299333 3827 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0329 02:51:51.299780 3827 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0329 02:51:51.299801 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0329 02:51:51.299813 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0329 02:51:51.301222 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0329 02:51:51.301243 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0329 02:51:51.301901 3827 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0329 02:51:51.301916 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0329 02:51:51.302580 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0329 02:51:51.302597 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0329 02:51:51.302605 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0329 02:51:51.303060 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0329 02:51:51.303074 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0329 02:51:51.303081 3827 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0329 02:51:51.303552 3827 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0329 02:51:51.303571 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0329 02:51:51.303591 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0329 02:51:51.304251 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0329 02:51:51.304270 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0329 02:51:51.304277 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0329 02:51:51.304826 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0329 02:51:51.304845 3827 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0329 02:51:51.304854 3827 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0329 02:51:51.306766 3827 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0329 02:51:51.306786 3827 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0329 02:51:51.306797 3827 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0329 02:51:51.307851 3827 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0329 02:51:51.307872 3827 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0329 02:51:51.307881 3827 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0329 02:51:51.308180 3827 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0329 02:51:51.308200 3827 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0329 02:51:51.308211 3827 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0329 02:51:51.309396 3827 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0329 02:51:51.309420 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0329 02:51:51.309431 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0329 02:51:51.309930 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0329 02:51:51.309950 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0329 02:51:51.309957 3827 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0329 02:51:51.310303 3827 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0329 02:51:51.310325 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0329 02:51:51.310334 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0329 02:51:51.311589 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0329 02:51:51.311608 3827 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0329 02:51:51.311615 3827 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0329 02:51:51.312102 3827 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0329 02:51:51.312123 3827 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0329 02:51:51.312133 3827 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0329 02:51:51.312407 3827 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0329 02:51:51.312425 3827 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0329 02:51:51.312434 3827 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0329 02:51:51.313452 3827 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0329 02:51:51.313473 3827 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0329 02:51:51.313485 3827 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0329 02:51:51.315217 3827 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0329 02:51:51.315238 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0329 02:51:51.315248 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0329 02:51:51.315773 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0329 02:51:51.315791 3827 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0329 02:51:51.315798 3827 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0329 02:51:51.316348 3827 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0329 02:51:51.316370 3827 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0329 02:51:51.316380 3827 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0329 02:51:51.316932 3827 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0329 02:51:51.316951 3827 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0329 02:51:51.316963 3827 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0329 02:51:51.318135 3827 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0329 02:51:51.318365 3827 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-mhrm8 of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0329 02:51:51.323429 3827 egressfirewall.go:319] Starting EgressFirewall controller I0329 02:51:51.323460 3827 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0329 02:51:51.323481 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0329 02:51:51.323491 3827 controller.go:133] Adding controller egress-firewall-controller event handlers I0329 02:51:51.323519 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0329 02:51:51.323531 3827 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0329 02:51:51.323493 3827 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0329 02:51:51.323784 3827 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0329 02:51:51.323797 3827 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0329 02:51:51.323820 3827 egressqos.go:193] Setting up event handlers for EgressQoS I0329 02:51:51.323900 3827 egressfirewall.go:628] Syncing node "ip-10-0-2-173.ec2.internal" for egress firewall I0329 02:51:51.323911 3827 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall I0329 02:51:51.323963 3827 egressqos.go:245] Starting EgressQoS Controller I0329 02:51:51.323977 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0329 02:51:51.323985 3827 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0329 02:51:51.323991 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0329 02:51:51.323995 3827 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0329 02:51:51.324000 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0329 02:51:51.324005 3827 shared_informer.go:356] "Caches are synced" controller="egressqos" I0329 02:51:51.324008 3827 egressqos.go:259] Repairing EgressQoSes I0329 02:51:51.324012 3827 egressqos.go:399] Starting repairing loop for egressqos I0329 02:51:51.324101 3827 egressqos.go:401] Finished repairing loop for egressqos: 88.329µs I0329 02:51:51.324117 3827 egressservice_zone.go:132] Setting up event handlers for Egress Services I0329 02:51:51.324140 3827 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-37.ec2.internal I0329 02:51:51.324186 3827 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-37.ec2.internal : 49.353µs I0329 02:51:51.324233 3827 egressservice_zone.go:210] Starting Egress Services Controller I0329 02:51:51.324243 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0329 02:51:51.324248 3827 shared_informer.go:356] "Caches are synced" controller="egressservices" I0329 02:51:51.324257 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0329 02:51:51.324264 3827 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0329 02:51:51.324270 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0329 02:51:51.324274 3827 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0329 02:51:51.324280 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0329 02:51:51.324284 3827 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0329 02:51:51.324287 3827 egressservice_zone.go:228] Repairing Egress Services I0329 02:51:51.324527 3827 master_controller.go:88] Starting Admin Policy Based Route Controller I0329 02:51:51.324539 3827 external_controller.go:278] Starting Admin Policy Based Route Controller I0329 02:51:51.324550 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:51.324560 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 10.985µs I0329 02:51:51.324567 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-173.ec2.internal I0329 02:51:51.324579 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-173.ec2.internal: 12.387µs I0329 02:51:51.324641 3827 default_network_controller.go:568] Completing all the Watchers took 311.550967ms I0329 02:51:51.324653 3827 default_network_controller.go:572] Starting unidling controllers I0329 02:51:51.324664 3827 unidle.go:46] Registering OVN SB ControllerEvent handler I0329 02:51:51.324674 3827 unidle.go:63] Populating Initial ContollerEvent events I0329 02:51:51.324691 3827 unidle.go:79] Setting up event handlers for services I0329 02:51:51.324787 3827 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0329 02:51:51.393779 3827 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0329 02:51:51.394616 3827 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 812.047µs I0329 02:51:51.450892 3827 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0329 02:51:51.549994 3827 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:52.049693 3827 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:52.197971 3827 namespace.go:98] [openshift-service-ca] adding namespace I0329 02:51:52.198871 3827 namespace.go:102] [openshift-service-ca] adding namespace took 878.31µs I0329 02:51:52.228827 3827 namespace.go:142] [openshift-service-ca] updating namespace I0329 02:51:52.550381 3827 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:52.785687 3827 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0329 02:51:53.050128 3827 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:53.065983 3827 bridgeconfig.go:537] Gateway is ready I0329 02:51:53.066002 3827 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0329 02:51:53.076688 3827 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.37. Updating OVN encap IP. I0329 02:51:53.081619 3827 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.37 - it is already configured I0329 02:51:53.081633 3827 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.37/24:{} 172.20.0.1/32:{}]. Updating annotations. I0329 02:51:53.081884 3827 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.37/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-37.ec2.internal","mac-address":"0a:ff:e4:b6:c5:ef","ip-addresses":["10.0.1.37/24"],"ip-address":"10.0.1.37/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:94753d28-646c-4b21-9f3b-9a0207ae18b0 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.37/24"}] on node ip-10-0-1-37.ec2.internal I0329 02:51:53.096934 3827 master.go:532] Adding or Updating local node "ip-10-0-1-37.ec2.internal" for network "default" I0329 02:51:53.097001 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:53.097008 3827 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall I0329 02:51:53.097020 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 24.467µs I0329 02:51:53.097076 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-37.ec2.internal in network "default" I0329 02:51:53.097125 3827 node_tracker.go:169] Node ip-10-0-1-37.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:53.097147 3827 services_controller.go:574] Full service sync requested for network=default W0329 02:51:53.097369 3827 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0329 02:51:53.097377 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0329 02:51:53.098793 3827 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.37 I0329 02:51:53.098916 3827 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-37.ec2.internal I0329 02:51:53.102435 3827 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-37.ec2.internal. Updating no re-route policies I0329 02:51:53.102524 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bwdh8 I0329 02:51:53.102543 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-7hs79 I0329 02:51:53.102560 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-w7gcs I0329 02:51:53.102563 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-k9snd I0329 02:51:53.102567 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-lh6lq I0329 02:51:53.102574 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-w7gcs I0329 02:51:53.102578 3827 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-zjsdb I0329 02:51:53.102571 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9 I0329 02:51:53.102588 3827 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-zjsdb I0329 02:51:53.102585 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-lh6lq I0329 02:51:53.102588 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-37.ec2.internal I0329 02:51:53.102593 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zghr9 I0329 02:51:53.102602 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-37.ec2.internal I0329 02:51:53.102595 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9 I0329 02:51:53.102605 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-w7gcs after 0 failed attempt(s) I0329 02:51:53.102607 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-v6wqn I0329 02:51:53.102613 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-lh6lq after 0 failed attempt(s) I0329 02:51:53.102613 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zghr9 I0329 02:51:53.102621 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-37.ec2.internal after 0 failed attempt(s) I0329 02:51:53.102623 3827 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-37.ec2.internal I0329 02:51:53.102624 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-v6wqn I0329 02:51:53.102622 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-ddvrx I0329 02:51:53.102632 3827 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-37.ec2.internal I0329 02:51:53.102639 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-ddvrx I0329 02:51:53.102640 3827 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-zz6r9] creating logical port openshift-network-diagnostics_network-check-target-zz6r9 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:53.102644 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zghr9 after 0 failed attempt(s) I0329 02:51:53.102647 3827 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-37.ec2.internal after 0 failed attempt(s) I0329 02:51:53.102649 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-v6wqn after 0 failed attempt(s) I0329 02:51:53.102564 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-7hs79 I0329 02:51:53.102657 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-ddvrx after 0 failed attempt(s) I0329 02:51:53.102684 3827 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-7hs79] creating logical port openshift-multus_network-metrics-daemon-7hs79 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:53.102579 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-k9snd I0329 02:51:53.102726 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-k9snd after 0 failed attempt(s) I0329 02:51:53.102550 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bwdh8 I0329 02:51:53.102753 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bwdh8 after 0 failed attempt(s) I0329 02:51:53.102546 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-jgvrt I0329 02:51:53.102769 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-jgvrt I0329 02:51:53.102786 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-jgvrt after 0 failed attempt(s) I0329 02:51:53.102611 3827 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-zjsdb after 0 failed attempt(s) I0329 02:51:53.103499 3827 pods.go:271] [openshift-network-diagnostics/network-check-target-zz6r9] addLogicalPort took 867.323µs, libovsdb time 328.587µs I0329 02:51:53.103516 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9 after 0 failed attempt(s) I0329 02:51:53.103619 3827 pods.go:271] [openshift-multus/network-metrics-daemon-7hs79] addLogicalPort took 942.399µs, libovsdb time 440.8µs I0329 02:51:53.103631 3827 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-7hs79 after 0 failed attempt(s) I0329 02:51:53.115683 3827 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0329 02:51:53.127794 3827 iptables.go:108] Creating table: mangle chain: OUTPUT I0329 02:51:53.131496 3827 iptables.go:108] Creating table: nat chain: OUTPUT I0329 02:51:53.153409 3827 iptables.go:108] Creating table: nat chain: PREROUTING I0329 02:51:53.209679 3827 iptables.go:108] Creating table: filter chain: FORWARD I0329 02:51:53.228054 3827 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0329 02:51:53.228299 3827 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0329 02:51:53.228355 3827 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0329 02:51:53.228447 3827 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0329 02:51:53.228516 3827 gateway_init.go:325] Gateway and management port readiness took 2.182301645s I0329 02:51:53.228550 3827 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0329 02:51:53.228596 3827 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0329 02:51:53.228612 3827 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0329 02:51:53.228627 3827 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0329 02:51:53.228704 3827 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0329 02:51:53.228789 3827 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.347µs I0329 02:51:53.229451 3827 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0329 02:51:53.229487 3827 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 23.471µs I0329 02:51:53.229497 3827 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 44.443µs I0329 02:51:53.229505 3827 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0329 02:51:53.229521 3827 nad_controller.go:173] [node-nad-controller NAD controller]: started I0329 02:51:53.229529 3827 default_node_network_controller.go:898] Starting the default node network controller I0329 02:51:53.229535 3827 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0329 02:51:53.229604 3827 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.115µs I0329 02:51:53.229690 3827 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0329 02:51:53.229700 3827 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0329 02:51:53.229729 3827 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000be6450 0xc00027be30 0xc00027bea0 0xc00027bf10 0xc0002ee000 0xc0002ee070 0xc0002ee0e0 0xc0000359f0 0xc0002ee150 0xc0002ee1c0 0xc000cb6280 0xc0063ff1f0 0xc0002ee310 map[0x2f67040:0xc002692fc0 0x2f67580:0xc002693140 0x2f68000:0xc002692840 0x2f682a0:0xc002692f60 0x2f687e0:0xc002693020 0x2f68a80:0xc002693080 0x2f68d20:0xc0026930e0 0x2f69500:0xc0026931a0 0x2f697a0:0xc002693200 0x2f93d40:0xc001232780 0x2f940e0:0xc0012327e0 0x2f94bc0:0xc000bdfaa0 0x2f96520:0xc001200a80 0x2f97000:0xc0028115c0 0x2f97e80:0xc001232660] 0xc00084a850 0} I0329 02:51:53.229792 3827 gateway.go:151] Starting gateway service sync I0329 02:51:53.230194 3827 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0329 02:51:53.232630 3827 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0329 02:51:53.242931 3827 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0329 02:51:53.244173 3827 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0329 02:51:53.245491 3827 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0329 02:51:53.279903 3827 gateway.go:168] Gateway service sync done. Time taken: 50.097057ms I0329 02:51:53.280392 3827 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30239 I0329 02:51:53.280755 3827 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000be6450 0xc00027be30 0xc00027bea0 0xc00027bf10 0xc0002ee000 0xc0002ee070 0xc0002ee0e0 0xc0000359f0 0xc0002ee150 0xc0002ee1c0 0xc000cb6280 0xc0063ff1f0 0xc0002ee310 map[0x2f67040:0xc002692fc0 0x2f67580:0xc002693140 0x2f68000:0xc002692840 0x2f682a0:0xc002692f60 0x2f687e0:0xc002693020 0x2f68a80:0xc002693080 0x2f68d20:0xc0026930e0 0x2f69500:0xc0026931a0 0x2f697a0:0xc002693200 0x2f93d40:0xc001232780 0x2f940e0:0xc0012327e0 0x2f94bc0:0xc000bdfaa0 0x2f96520:0xc001200a80 0x2f97000:0xc0028115c0 0x2f97e80:0xc001232660] 0xc00084a850 0} I0329 02:51:53.281305 3827 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000be6450 0xc00027be30 0xc00027bea0 0xc00027bf10 0xc0002ee000 0xc0002ee070 0xc0002ee0e0 0xc0000359f0 0xc0002ee150 0xc0002ee1c0 0xc000cb6280 0xc0063ff1f0 0xc0002ee310 map[0x2f67040:0xc002692fc0 0x2f67580:0xc002693140 0x2f68000:0xc002692840 0x2f682a0:0xc002692f60 0x2f687e0:0xc002693020 0x2f68a80:0xc002693080 0x2f68d20:0xc0026930e0 0x2f69500:0xc0026931a0 0x2f697a0:0xc002693200 0x2f93d40:0xc001232780 0x2f940e0:0xc0012327e0 0x2f94bc0:0xc000bdfaa0 0x2f96520:0xc001200a80 0x2f97000:0xc0028115c0 0x2f97e80:0xc001232660] 0xc00084a850 0} I0329 02:51:53.281349 3827 gateway.go:519] Reconciling gateway with updates I0329 02:51:53.287120 3827 gateway.go:326] Spawning Conntrack Rule Check Thread I0329 02:51:53.287163 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:53.287177 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:53.287181 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:53.287191 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:53.287194 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:53.287202 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:53.287213 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:53.287214 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:53.287225 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:53.287229 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:53.287233 3827 port_claim.go:71] Svc openshift-ingress/router-default: port 30780 is already open, no action needed I0329 02:51:53.287236 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:53.287235 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0329 02:51:53.287251 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:53.287264 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0329 02:51:53.287266 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:53.287272 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:53.287283 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0329 02:51:53.287167 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:53.287301 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:53.287306 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:53.287308 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:53.287198 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:53.287314 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:53.287318 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0329 02:51:53.287322 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:53.287331 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:53.287336 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:53.287333 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:53.287325 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:53.287305 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:53.287349 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0329 02:51:53.287212 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:53.287359 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:53.287370 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0329 02:51:53.287285 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:53.287373 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:53.287384 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:53.287285 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0329 02:51:53.287383 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:53.287386 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0329 02:51:53.287392 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0329 02:51:53.287400 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0329 02:51:53.287401 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:53.287256 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0329 02:51:53.287409 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0329 02:51:53.287408 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0329 02:51:53.287406 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:53.287418 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:53.287417 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0329 02:51:53.287234 3827 default_node_network_controller.go:943] Gateway and management port readiness took 57.524438ms I0329 02:51:53.287430 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:53.287430 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:53.287430 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:53.287442 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:53.287443 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:53.287288 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0329 02:51:53.287453 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0329 02:51:53.287454 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0329 02:51:53.287304 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:53.287469 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:53.287472 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:53.287478 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0329 02:51:53.287446 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:53.287486 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:53.287493 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:53.287500 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:53.287459 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0329 02:51:53.287535 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0329 02:51:53.287536 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:53.287480 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:53.287547 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:53.287551 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:53.287297 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:53.287560 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0329 02:51:53.287300 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:53.287575 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:53.287570 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:53.287487 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:53.287602 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:53.287314 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:53.287615 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0329 02:51:53.287311 3827 port_claim.go:71] Svc openshift-ingress/router-default: port 30101 is already open, no action needed I0329 02:51:53.287316 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:53.287700 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:53.287354 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:53.287270 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:53.287841 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:53.287359 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:53.287862 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:53.287361 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0329 02:51:53.287182 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:53.287922 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:53.287369 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0329 02:51:53.287373 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0329 02:51:53.287386 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:53.287957 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:53.287391 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:53.287409 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:53.288101 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:53.287420 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:53.288196 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:53.288223 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0329 02:51:53.287291 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:53.288239 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:53.287295 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:53.288356 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:53.287526 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:53.287314 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:53.288476 3827 node_ip_handler_linux.go:160] Node IP manager is running I0329 02:51:53.288483 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:53.288487 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary W0329 02:51:53.327302 3827 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0329 02:51:53.327535 3827 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0329 02:51:53.327743 3827 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0329 02:51:53.337470 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0329 02:51:53.337498 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0329 02:51:53.337506 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0329 02:51:53.337513 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0329 02:51:53.337520 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0329 02:51:53.337525 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0329 02:51:53.378075 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0329 02:51:53.378105 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0329 02:51:53.378111 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0329 02:51:53.378119 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0329 02:51:53.417592 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0329 02:51:53.417601 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0329 02:51:53.417622 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0329 02:51:53.417629 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0329 02:51:53.417640 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0329 02:51:53.417645 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0329 02:51:53.417651 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0329 02:51:53.417657 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0329 02:51:53.417663 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0329 02:51:53.417670 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0329 02:51:53.417675 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0329 02:51:53.417682 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0329 02:51:53.417688 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0329 02:51:53.417695 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0329 02:51:55.341997 3827 default_node_network_controller.go:1364] Starting node controller node sync W0329 02:51:55.342028 3827 egressip_healthcheck.go:76] Health checking using insecure connection I0329 02:51:55.342405 3827 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0329 02:51:55.354676 3827 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 12.410428ms I0329 02:51:55.354856 3827 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-173.ec2.internal", IP: 10.0.2.173 to PMTUD blocking rules I0329 02:51:55.354872 3827 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-173.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0329 02:51:55.360794 3827 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0329 02:51:55.361092 3827 egressservice_node.go:93] Setting up event handlers for Egress Services I0329 02:51:55.361299 3827 egressservice_node.go:183] Starting Egress Services Controller I0329 02:51:55.361320 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0329 02:51:55.361329 3827 shared_informer.go:356] "Caches are synced" controller="egressservices" I0329 02:51:55.361354 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0329 02:51:55.361364 3827 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0329 02:51:55.361373 3827 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0329 02:51:55.361380 3827 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0329 02:51:55.361386 3827 egressservice_node.go:197] Repairing Egress Services I0329 02:51:55.370822 3827 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0329 02:51:55.370834 3827 external_controller.go:278] Starting Admin Policy Based Route Controller I0329 02:51:55.371060 3827 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0329 02:51:55.371073 3827 link_network_manager.go:117] Link manager is running I0329 02:51:55.371084 3827 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0329 02:51:55.371133 3827 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0329 02:51:55.371472 3827 vrf_manager.go:128] VRF manager is running I0329 02:51:55.371561 3827 node_controller_manager.go:437] Removing flows to drop GARP I0329 02:51:55.371573 3827 gateway.go:519] Reconciling gateway with updates I0329 02:51:55.396296 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:55.396318 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:55.396302 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:55.396327 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:55.396330 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:55.396318 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:55.396329 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:55.396343 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:55.396345 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:55.396347 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0329 02:51:55.396348 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:55.396354 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:55.396362 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:55.396356 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:55.396378 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:55.396381 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:55.396387 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:55.396393 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:55.396390 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:55.396397 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:55.396401 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:55.396402 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:55.396405 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:55.396408 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:55.396400 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:55.396410 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:55.396402 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:55.396416 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:55.396422 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:55.396427 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:55.396427 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:55.396433 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:55.396429 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:55.396438 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0329 02:51:55.396439 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:55.396443 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:55.396441 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:55.396451 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396453 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:55.396303 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:55.396461 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:55.396465 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0329 02:51:55.396391 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:55.396407 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396472 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0329 02:51:55.396467 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:55.396490 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0329 02:51:55.396493 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0329 02:51:55.396496 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:55.396341 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0329 02:51:55.396505 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:55.396504 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0329 02:51:55.396509 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0329 02:51:55.396504 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0329 02:51:55.396513 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0329 02:51:55.396513 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0329 02:51:55.396449 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396332 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0329 02:51:55.396528 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0329 02:51:55.396532 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0329 02:51:55.396534 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0329 02:51:55.396359 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:55.396548 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:55.396554 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0329 02:51:55.396322 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:55.396560 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0329 02:51:55.396349 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0329 02:51:55.396505 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396306 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:55.396641 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:55.396666 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396335 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:55.396685 3827 port_claim.go:71] Svc openshift-ingress/router-default: port 30780 is already open, no action needed I0329 02:51:55.396693 3827 port_claim.go:71] Svc openshift-ingress/router-default: port 30101 is already open, no action needed I0329 02:51:55.396333 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:55.396743 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:55.396747 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396338 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:55.396337 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:55.396774 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:55.396319 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:55.396814 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:55.396351 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:55.396839 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:55.396362 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:55.396863 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:55.396867 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0329 02:51:55.396362 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0329 02:51:55.396370 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:55.396897 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:55.396369 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:55.396924 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:55.396931 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0329 02:51:55.396372 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:55.396941 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0329 02:51:55.396371 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:55.396958 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:55.396966 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0329 02:51:55.396379 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:55.396976 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:55.396383 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:55.397010 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:55.396382 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:55.397052 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:55.396390 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:55.396393 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:55.397093 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:55.396414 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:55.396410 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:55.397136 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:55.396426 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:55.396466 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0329 02:51:55.396478 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:55.396518 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0329 02:51:55.396584 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0329 02:51:55.396320 3827 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:55.397219 3827 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:55.418745 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0329 02:51:55.418755 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0329 02:51:55.418771 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0329 02:51:55.460935 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0329 02:51:55.460967 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0329 02:51:55.500009 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0329 02:51:55.500056 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0329 02:51:55.500067 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0329 02:51:55.500081 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0329 02:51:55.500092 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0329 02:51:55.500099 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0329 02:51:55.500105 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0329 02:51:55.500112 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0329 02:51:55.500117 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0329 02:51:55.500126 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0329 02:51:55.500135 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0329 02:51:55.500149 3827 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0329 02:51:56.597079 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:56.597107 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 34.818µs I0329 02:51:56.626853 3827 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c558b4c9b-jdfv5] creating logical port openshift-image-registry_image-registry-7c558b4c9b-jdfv5 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:56.626977 3827 kube.go:256] Updating pod openshift-image-registry/image-registry-7c558b4c9b-jdfv5 I0329 02:51:56.653711 3827 pod.go:62] [openshift-image-registry/image-registry-7c558b4c9b-jdfv5] pod update took 26.764668ms I0329 02:51:56.653730 3827 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7c558b4c9b-jdfv5] addLogicalPort annotation time took 26.789098ms I0329 02:51:56.654870 3827 pods.go:271] [openshift-image-registry/image-registry-7c558b4c9b-jdfv5] addLogicalPort took 28.032751ms, libovsdb time 847.623µs I0329 02:51:56.665732 3827 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-84747c8b65-gtsnv] creating logical port openshift-ingress_router-default-84747c8b65-gtsnv for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:56.665848 3827 kube.go:256] Updating pod openshift-ingress/router-default-84747c8b65-gtsnv I0329 02:51:56.668693 3827 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-qg5f8] creating logical port openshift-ingress-canary_ingress-canary-qg5f8 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:56.668781 3827 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-qg5f8 I0329 02:51:56.695070 3827 pod.go:62] [openshift-ingress-canary/ingress-canary-qg5f8] pod update took 26.316999ms I0329 02:51:56.695087 3827 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-qg5f8] addLogicalPort annotation time took 26.337047ms I0329 02:51:56.695997 3827 pods.go:271] [openshift-ingress-canary/ingress-canary-qg5f8] addLogicalPort took 27.316025ms, libovsdb time 697.849µs I0329 02:51:56.696269 3827 pod.go:62] [openshift-ingress/router-default-84747c8b65-gtsnv] pod update took 30.466026ms I0329 02:51:56.696284 3827 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-84747c8b65-gtsnv] addLogicalPort annotation time took 30.486417ms I0329 02:51:56.697099 3827 pods.go:271] [openshift-ingress/router-default-84747c8b65-gtsnv] addLogicalPort took 31.379582ms, libovsdb time 630.281µs I0329 02:51:56.697141 3827 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-mwzkc] creating logical port openshift-dns_dns-default-mwzkc for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:56.697186 3827 kube.go:256] Updating pod openshift-dns/dns-default-mwzkc I0329 02:51:56.713615 3827 pod.go:62] [openshift-dns/dns-default-mwzkc] pod update took 16.442891ms I0329 02:51:56.713637 3827 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-mwzkc] addLogicalPort annotation time took 16.464969ms I0329 02:51:56.714567 3827 pods.go:271] [openshift-dns/dns-default-mwzkc] addLogicalPort took 17.42907ms, libovsdb time 650.85µs I0329 02:51:56.928522 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:51:56.928543 3827 node_tracker.go:220] Node ip-10-0-0-4.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.928529 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:51:56.928559 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 33.099µs I0329 02:51:56.928587 3827 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall E0329 02:51:56.928612 3827 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.928635 3827 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-4.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-4.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-4.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-4.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-4.ec2.internal, error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] I0329 02:51:56.955629 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0329 02:51:56.955662 3827 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input E0329 02:51:56.955673 3827 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.969552 3827 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall I0329 02:51:56.969595 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:56.969620 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 30.388µs I0329 02:51:56.972555 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0329 02:51:56.972591 3827 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.972608 3827 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall E0329 02:51:56.972612 3827 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input I0329 02:51:56.972625 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:51:56.972634 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 12.868µs E0329 02:51:57.076626 3827 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input E0329 02:51:57.076703 3827 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input I0329 02:51:57.076730 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0329 02:51:57.076804 3827 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-4.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-4.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-4.ec2.internal" I0329 02:51:57.100549 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0329 02:51:57.194652 3827 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:51:57.194696 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:51:57.194713 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 20.651µs I0329 02:51:57.194729 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:57.241054 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:57.334936 3827 cni.go:403] [openshift-ingress-canary/ingress-canary-qg5f8 0a86a607ae723c1a15dc03db9b61a7d1ed7b531a3bd9898a2fe962316d33279a network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-qg5f8 0a86a607ae723c1a15dc03db9b61a7d1ed7b531a3bd9898a2fe962316d33279a network default NAD default] I0329 02:51:57.350857 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-qg5f8, hostIfaceName: 0a86a607ae723c1, network: default, NAD default, SandboxID: "0a86a607ae723c1a15dc03db9b61a7d1ed7b531a3bd9898a2fe962316d33279a", PCI device ID: , UID: "4653e9bd-cb67-4b8c-8311-4e51b8fa87a3", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0329 02:51:57.389990 3827 cni.go:424] [openshift-ingress-canary/ingress-canary-qg5f8 0a86a607ae723c1a15dc03db9b61a7d1ed7b531a3bd9898a2fe962316d33279a network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-qg5f8 0a86a607ae723c1a15dc03db9b61a7d1ed7b531a3bd9898a2fe962316d33279a network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:54:b0:7a:a3:b9\",\"name\":\"0a86a607ae723c1\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a51caaec-b616-4a8d-8749-14a4a9a850d1\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:51:57.422766 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:57.466447 3827 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-clldw] creating logical port kube-system_global-pull-secret-syncer-clldw for pod on switch ip-10-0-1-37.ec2.internal I0329 02:51:57.466569 3827 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-clldw I0329 02:51:57.482198 3827 pod.go:62] [kube-system/global-pull-secret-syncer-clldw] pod update took 15.653135ms I0329 02:51:57.482224 3827 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-clldw] addLogicalPort annotation time took 15.684686ms I0329 02:51:57.483464 3827 pods.go:271] [kube-system/global-pull-secret-syncer-clldw] addLogicalPort took 17.03163ms, libovsdb time 898.936µs I0329 02:51:57.822201 3827 cni.go:403] [kube-system/global-pull-secret-syncer-clldw 3fa67df0ef9aa60cb4cab2afe789c32ba32529d8a27aad2ee1b5e87d5cc83473 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-clldw 3fa67df0ef9aa60cb4cab2afe789c32ba32529d8a27aad2ee1b5e87d5cc83473 network default NAD default] I0329 02:51:57.831202 3827 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-clldw, hostIfaceName: 3fa67df0ef9aa60, network: default, NAD default, SandboxID: "3fa67df0ef9aa60cb4cab2afe789c32ba32529d8a27aad2ee1b5e87d5cc83473", PCI device ID: , UID: "e3a122f8-be08-409d-ac60-823fc0057872", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0329 02:51:57.870686 3827 cni.go:424] [kube-system/global-pull-secret-syncer-clldw 3fa67df0ef9aa60cb4cab2afe789c32ba32529d8a27aad2ee1b5e87d5cc83473 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-clldw 3fa67df0ef9aa60cb4cab2afe789c32ba32529d8a27aad2ee1b5e87d5cc83473 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:b4:13:be:06:1f\",\"name\":\"3fa67df0ef9aa60\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be11999f-e681-4350-b914-50c50ea100da\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:51:59.364966 3827 cni.go:403] [openshift-network-diagnostics/network-check-target-zz6r9 e30d05d634c88e982e7080febcd17b5116cd29760abad4194236021ec4813a08 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-zz6r9 e30d05d634c88e982e7080febcd17b5116cd29760abad4194236021ec4813a08 network default NAD default] I0329 02:51:59.374594 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-zz6r9, hostIfaceName: e30d05d634c88e9, network: default, NAD default, SandboxID: "e30d05d634c88e982e7080febcd17b5116cd29760abad4194236021ec4813a08", PCI device ID: , UID: "68a23aab-4596-4f1b-8447-881e5e94962d", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0329 02:51:59.420588 3827 cni.go:424] [openshift-network-diagnostics/network-check-target-zz6r9 e30d05d634c88e982e7080febcd17b5116cd29760abad4194236021ec4813a08 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-zz6r9 e30d05d634c88e982e7080febcd17b5116cd29760abad4194236021ec4813a08 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:cf:ea:e7:d7:e4\",\"name\":\"e30d05d634c88e9\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4272c929-6873-4970-b349-343d0aac9f91\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 02:51:59.582220 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0329 02:51:59.957675 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:52:00.364880 3827 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-7794f78cd7-7jq8s IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-7794f78cd7-7jq8s: no pod IPs found I0329 02:52:01.747094 3827 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:52:01.747842 3827 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:52:01.766517 3827 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:01.766533 3827 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:01.775409 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c558b4c9b-8rqfz I0329 02:52:01.775427 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c558b4c9b-8rqfz W0329 02:52:02.761150 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:02.784742 3827 cni.go:403] [openshift-dns/dns-default-mwzkc 689af1a3fc9ffbc2898d815991a048611077b9bbdea1521228958d218482125a network default NAD default] ADD starting CNI request [openshift-dns/dns-default-mwzkc 689af1a3fc9ffbc2898d815991a048611077b9bbdea1521228958d218482125a network default NAD default] I0329 02:52:02.785255 3827 cni.go:403] [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default] I0329 02:52:02.800330 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7c558b4c9b-jdfv5, hostIfaceName: 0e9b16c25d7110f, network: default, NAD default, SandboxID: "0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9", PCI device ID: , UID: "4ff3971f-0994-44ac-b96c-af2ca08d2ee3", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0329 02:52:02.801015 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-mwzkc, hostIfaceName: 689af1a3fc9ffbc, network: default, NAD default, SandboxID: "689af1a3fc9ffbc2898d815991a048611077b9bbdea1521228958d218482125a", PCI device ID: , UID: "60198e6f-f0d7-4698-8132-fdeb01ae0507", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0329 02:52:02.846190 3827 cni.go:424] [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:c2:0c:ee:ce:3f\",\"name\":\"0e9b16c25d7110f\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d5369fde-46f6-4f84-b39c-f333d204564d\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:02.862523 3827 cni.go:424] [openshift-dns/dns-default-mwzkc 689af1a3fc9ffbc2898d815991a048611077b9bbdea1521228958d218482125a network default NAD default] ADD finished CNI request [openshift-dns/dns-default-mwzkc 689af1a3fc9ffbc2898d815991a048611077b9bbdea1521228958d218482125a network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:d5:26:31:2c:18\",\"name\":\"689af1a3fc9ffbc\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9e8a3b7-c427-496d-8ac7-2812384860af\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 02:52:03.488255 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:03.511451 3827 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0329 02:52:04.559828 3827 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-5gqrp] creating logical port openshift-insights_insights-runtime-extractor-5gqrp for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:04.559922 3827 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-5gqrp I0329 02:52:04.574605 3827 pod.go:62] [openshift-insights/insights-runtime-extractor-5gqrp] pod update took 14.708584ms I0329 02:52:04.574627 3827 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-5gqrp] addLogicalPort annotation time took 14.736171ms I0329 02:52:04.575660 3827 pods.go:271] [openshift-insights/insights-runtime-extractor-5gqrp] addLogicalPort took 15.850131ms, libovsdb time 757.099µs W0329 02:52:05.476059 3827 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:05.476080 3827 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:05.476087 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:05.567235 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:05.570342 3827 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-64fc4f6495-sbb2r IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-64fc4f6495-sbb2r: no pod IPs found I0329 02:52:06.456560 3827 cni.go:403] [openshift-insights/insights-runtime-extractor-5gqrp 0cb5717a3d15349963866e98556b1aa76f08008dd14831b06c9fce5c2bc2b9bb network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-5gqrp 0cb5717a3d15349963866e98556b1aa76f08008dd14831b06c9fce5c2bc2b9bb network default NAD default] I0329 02:52:06.465372 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-5gqrp, hostIfaceName: 0cb5717a3d15349, network: default, NAD default, SandboxID: "0cb5717a3d15349963866e98556b1aa76f08008dd14831b06c9fce5c2bc2b9bb", PCI device ID: , UID: "2deab760-4de4-42d6-a315-c59a267a124d", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0329 02:52:06.506818 3827 cni.go:424] [openshift-insights/insights-runtime-extractor-5gqrp 0cb5717a3d15349963866e98556b1aa76f08008dd14831b06c9fce5c2bc2b9bb network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-5gqrp 0cb5717a3d15349963866e98556b1aa76f08008dd14831b06c9fce5c2bc2b9bb network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:25:a1:f7:e9:9d\",\"name\":\"0cb5717a3d15349\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d5a4ab5-e2c8-4cfb-87ba-3e000f1c4dc7\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:07.043892 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:08.569811 3827 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-kfvs6] creating logical port openshift-console_downloads-56d45c498b-kfvs6 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:08.569914 3827 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-kfvs6 I0329 02:52:08.589297 3827 pod.go:62] [openshift-console/downloads-56d45c498b-kfvs6] pod update took 19.418544ms I0329 02:52:08.589324 3827 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-kfvs6] addLogicalPort annotation time took 19.447097ms I0329 02:52:08.590416 3827 pods.go:271] [openshift-console/downloads-56d45c498b-kfvs6] addLogicalPort took 20.619865ms, libovsdb time 795.806µs W0329 02:52:08.854795 3827 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:08.854817 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:08.854822 3827 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0329 02:52:08.983515 3827 cni.go:403] [openshift-console/downloads-56d45c498b-kfvs6 9cf685918f5bdcbf0914ad1f846e09c626d16e3f87a2b644dd4f4ee23efa5859 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-kfvs6 9cf685918f5bdcbf0914ad1f846e09c626d16e3f87a2b644dd4f4ee23efa5859 network default NAD default] I0329 02:52:08.993792 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-kfvs6, hostIfaceName: 9cf685918f5bdcb, network: default, NAD default, SandboxID: "9cf685918f5bdcbf0914ad1f846e09c626d16e3f87a2b644dd4f4ee23efa5859", PCI device ID: , UID: "82c28a85-bc6c-473f-b8b1-cd6fe3927935", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0329 02:52:09.037260 3827 cni.go:424] [openshift-console/downloads-56d45c498b-kfvs6 9cf685918f5bdcbf0914ad1f846e09c626d16e3f87a2b644dd4f4ee23efa5859 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-kfvs6 9cf685918f5bdcbf0914ad1f846e09c626d16e3f87a2b644dd4f4ee23efa5859 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:7a:18:0f:10:5d\",\"name\":\"9cf685918f5bdcb\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e73c3a9-fb4e-4091-b206-484ae8601b00\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:12.645409 3827 cni.go:403] [openshift-ingress/router-default-84747c8b65-gtsnv ae1a2410851839975ee3272d590bc8e94d1f001b71cb4146c05bb767fe58bf99 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-84747c8b65-gtsnv ae1a2410851839975ee3272d590bc8e94d1f001b71cb4146c05bb767fe58bf99 network default NAD default] I0329 02:52:12.666457 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-84747c8b65-gtsnv, hostIfaceName: ae1a24108518399, network: default, NAD default, SandboxID: "ae1a2410851839975ee3272d590bc8e94d1f001b71cb4146c05bb767fe58bf99", PCI device ID: , UID: "848b61ea-b338-46e6-bb9a-f5b052c242ec", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0329 02:52:12.710968 3827 cni.go:424] [openshift-ingress/router-default-84747c8b65-gtsnv ae1a2410851839975ee3272d590bc8e94d1f001b71cb4146c05bb767fe58bf99 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-84747c8b65-gtsnv ae1a2410851839975ee3272d590bc8e94d1f001b71cb4146c05bb767fe58bf99 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:cb:9b:ef:ef:13\",\"name\":\"ae1a24108518399\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e313ff7f-30d5-4c77-89e0-2225dd0a1fc5\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:16.854744 3827 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:52:16.854766 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:52:16.854783 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 21.996µs I0329 02:52:16.854798 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:21.061706 3827 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-0-4.ec2.internal I0329 02:52:21.061740 3827 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-0-4.ec2.internal I0329 02:52:21.061842 3827 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-0-4.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-4.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-4.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-4.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-4.ec2.internal I0329 02:52:23.103840 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xxzzq I0329 02:52:23.103872 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xxzzq I0329 02:52:23.103837 3827 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-wwhgw I0329 02:52:23.103926 3827 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-xxzzq, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-xxzzq: suppressed error logged: pod openshift-network-diagnostics/network-check-target-xxzzq: no pod IPs found I0329 02:52:23.103966 3827 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-wwhgw I0329 02:52:23.104001 3827 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-wwhgw, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-wwhgw: suppressed error logged: pod openshift-multus/network-metrics-daemon-wwhgw: no pod IPs found I0329 02:52:30.028074 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:52:30.038776 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0329 02:52:30.038802 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0329 02:52:30.056158 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:30.056175 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:30.056180 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0329 02:52:30.098423 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:30.163629 3827 base_network_controller_pods.go:487] [default/openshift-console/console-7677b9b887-frl82] creating logical port openshift-console_console-7677b9b887-frl82 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:30.163725 3827 kube.go:256] Updating pod openshift-console/console-7677b9b887-frl82 I0329 02:52:30.186108 3827 pod.go:62] [openshift-console/console-7677b9b887-frl82] pod update took 22.40844ms I0329 02:52:30.186133 3827 base_network_controller_pods.go:937] [default/openshift-console/console-7677b9b887-frl82] addLogicalPort annotation time took 22.434644ms I0329 02:52:30.187365 3827 pods.go:271] [openshift-console/console-7677b9b887-frl82] addLogicalPort took 23.754917ms, libovsdb time 856.713µs I0329 02:52:30.208021 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-zl4mw for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:30.208157 3827 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw I0329 02:52:30.221008 3827 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0329 02:52:30.241700 3827 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw] pod update took 33.563045ms I0329 02:52:30.241723 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw] addLogicalPort annotation time took 33.59236ms I0329 02:52:30.243171 3827 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw] addLogicalPort took 35.167023ms, libovsdb time 1.148697ms I0329 02:52:30.586910 3827 cni.go:403] [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default] ADD starting CNI request [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default] I0329 02:52:30.596710 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7677b9b887-frl82, hostIfaceName: a126cdfb37dda28, network: default, NAD default, SandboxID: "a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0", PCI device ID: , UID: "4f510e02-9448-4dc2-b78a-ee5462da281e", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0329 02:52:30.626208 3827 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw e0595369d5021edebf64011881ac9d2184fe69a1fa53e98fc78488ed1f1a08b6 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw e0595369d5021edebf64011881ac9d2184fe69a1fa53e98fc78488ed1f1a08b6 network default NAD default] I0329 02:52:30.642462 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-zl4mw, hostIfaceName: e0595369d5021ed, network: default, NAD default, SandboxID: "e0595369d5021edebf64011881ac9d2184fe69a1fa53e98fc78488ed1f1a08b6", PCI device ID: , UID: "53976179-26b8-4d40-9df6-9b9b0cd019f5", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0329 02:52:30.643256 3827 cni.go:424] [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default] ADD finished CNI request [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:93:30:bb:d2:51\",\"name\":\"a126cdfb37dda28\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb9498db-2498-4109-a4a5-c7161ec97907\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:30.685207 3827 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw e0595369d5021edebf64011881ac9d2184fe69a1fa53e98fc78488ed1f1a08b6 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zl4mw e0595369d5021edebf64011881ac9d2184fe69a1fa53e98fc78488ed1f1a08b6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:5b:bd:3e:1c:6f\",\"name\":\"e0595369d5021ed\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9d61cc8-b60a-4512-8814-5eebfeb94b0f\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:30.753622 3827 cni.go:403] [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default] I0329 02:52:30.819818 3827 cni.go:424] [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-7c558b4c9b-jdfv5 0e9b16c25d7110f08ff22f652b109bebfc72a3f2030a1ee477137529446d47d9 network default NAD default], result "{}", err I0329 02:52:30.883078 3827 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:52:30.883121 3827 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7c558b4c9b-jdfv5 I0329 02:52:30.884603 3827 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7c558b4c9b-jdfv5, ips: 10.129.0.5 I0329 02:52:30.884633 3827 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0329 02:52:30.894761 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:30.895377 3827 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-78566ddd85-66r6s] creating logical port openshift-image-registry_image-registry-78566ddd85-66r6s for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:30.895475 3827 kube.go:256] Updating pod openshift-image-registry/image-registry-78566ddd85-66r6s I0329 02:52:30.915259 3827 pod.go:62] [openshift-image-registry/image-registry-78566ddd85-66r6s] pod update took 19.820779ms I0329 02:52:30.915287 3827 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-78566ddd85-66r6s] addLogicalPort annotation time took 19.850246ms I0329 02:52:30.916363 3827 pods.go:271] [openshift-image-registry/image-registry-78566ddd85-66r6s] addLogicalPort took 20.998976ms, libovsdb time 797.689µs I0329 02:52:31.051704 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:52:31.051757 3827 node_tracker.go:235] Node ip-10-0-0-4.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-4.ec2.internal" I0329 02:52:31.051756 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "ip-10-0-0-4.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:31.051776 3827 node_tracker.go:169] Node ip-10-0-0-4.ec2.internal switch + router changed, syncing services in network "default" I0329 02:52:31.051790 3827 services_controller.go:574] Full service sync requested for network=default W0329 02:52:31.052168 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.206304 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:52:31.206372 3827 node_tracker.go:169] Node ip-10-0-0-4.ec2.internal switch + router changed, syncing services in network "default" I0329 02:52:31.206388 3827 services_controller.go:574] Full service sync requested for network=default I0329 02:52:31.206392 3827 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "ip-10-0-0-4.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:52:31.206654 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.207185 3827 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-4.ec2.internal for the network default I0329 02:52:31.208425 3827 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-4.ec2.internal" on network "default" took: 1.251114ms I0329 02:52:31.268188 3827 cni.go:403] [openshift-multus/network-metrics-daemon-7hs79 1fd9d8cf30e042467732223dc0b699ccb3a2b6cec2473520907e7898da6f355b network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-7hs79 1fd9d8cf30e042467732223dc0b699ccb3a2b6cec2473520907e7898da6f355b network default NAD default] I0329 02:52:31.277104 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-7hs79, hostIfaceName: 1fd9d8cf30e0424, network: default, NAD default, SandboxID: "1fd9d8cf30e042467732223dc0b699ccb3a2b6cec2473520907e7898da6f355b", PCI device ID: , UID: "91161079-e64f-4adf-b960-97bd659eb6d2", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0329 02:52:31.322663 3827 cni.go:424] [openshift-multus/network-metrics-daemon-7hs79 1fd9d8cf30e042467732223dc0b699ccb3a2b6cec2473520907e7898da6f355b network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-7hs79 1fd9d8cf30e042467732223dc0b699ccb3a2b6cec2473520907e7898da6f355b network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:47:4a:8d:8c:bf\",\"name\":\"1fd9d8cf30e0424\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/98bf7e58-8383-4460-b149-717a4fb824af\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:31.327484 3827 cni.go:403] [openshift-image-registry/image-registry-78566ddd85-66r6s d870b0c7f934a8d738e498769f2bd00e324a223c3d0760dd9b32053ea4eb9ebe network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-78566ddd85-66r6s d870b0c7f934a8d738e498769f2bd00e324a223c3d0760dd9b32053ea4eb9ebe network default NAD default] I0329 02:52:31.336392 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-78566ddd85-66r6s, hostIfaceName: d870b0c7f934a8d, network: default, NAD default, SandboxID: "d870b0c7f934a8d738e498769f2bd00e324a223c3d0760dd9b32053ea4eb9ebe", PCI device ID: , UID: "0ba66307-4b6c-4f78-9a1f-26246f4b4ba7", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0329 02:52:31.385154 3827 cni.go:424] [openshift-image-registry/image-registry-78566ddd85-66r6s d870b0c7f934a8d738e498769f2bd00e324a223c3d0760dd9b32053ea4eb9ebe network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-78566ddd85-66r6s d870b0c7f934a8d738e498769f2bd00e324a223c3d0760dd9b32053ea4eb9ebe network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:d5:64:0a:e5:b3\",\"name\":\"d870b0c7f934a8d\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b4f804b7-3d2d-4640-b707-59489182ae25\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:31.558935 3827 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:31.558967 3827 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:31.569737 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c558b4c9b-jdfv5 I0329 02:52:31.569768 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c558b4c9b-jdfv5 I0329 02:52:31.757189 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:52:31.757204 3827 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:52:31.757215 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 34.073µs I0329 02:52:31.757184 3827 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:52:31.757281 3827 node_tracker.go:169] Node ip-10-0-0-4.ec2.internal switch + router changed, syncing services in network "default" I0329 02:52:31.757301 3827 services_controller.go:574] Full service sync requested for network=default I0329 02:52:31.757309 3827 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 10.0.0.4 to PMTUD blocking rules I0329 02:52:31.757331 3827 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0329 02:52:31.757842 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.764573 3827 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-4.ec2.internal. Updating no re-route policies I0329 02:52:31.775569 3827 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 10.0.0.4 to PMTUD blocking rules I0329 02:52:31.775586 3827 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0329 02:52:32.584838 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0329 02:52:32.616006 3827 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:32.637872 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0329 02:52:33.783590 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:33.783617 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:33.886467 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-c56ff685-sbksr] creating logical port openshift-monitoring_prometheus-operator-c56ff685-sbksr for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:33.886564 3827 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-c56ff685-sbksr I0329 02:52:33.902784 3827 pod.go:62] [openshift-monitoring/prometheus-operator-c56ff685-sbksr] pod update took 16.241047ms I0329 02:52:33.902865 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-c56ff685-sbksr] addLogicalPort annotation time took 16.326721ms I0329 02:52:33.904139 3827 pods.go:271] [openshift-monitoring/prometheus-operator-c56ff685-sbksr] addLogicalPort took 17.695772ms, libovsdb time 925.869µs W0329 02:52:33.936541 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0329 02:52:34.466516 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0329 02:52:34.597992 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0329 02:52:34.870729 3827 cni.go:403] [openshift-monitoring/prometheus-operator-c56ff685-sbksr 12d6ae9f5dd020fac415dc798d5189697c2beb175f7b11160e1820ab3cc3c811 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-c56ff685-sbksr 12d6ae9f5dd020fac415dc798d5189697c2beb175f7b11160e1820ab3cc3c811 network default NAD default] I0329 02:52:34.879753 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-c56ff685-sbksr, hostIfaceName: 12d6ae9f5dd020f, network: default, NAD default, SandboxID: "12d6ae9f5dd020fac415dc798d5189697c2beb175f7b11160e1820ab3cc3c811", PCI device ID: , UID: "52113e01-4c6d-42da-b7ea-9afe06c43678", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0329 02:52:34.920234 3827 cni.go:424] [openshift-monitoring/prometheus-operator-c56ff685-sbksr 12d6ae9f5dd020fac415dc798d5189697c2beb175f7b11160e1820ab3cc3c811 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-c56ff685-sbksr 12d6ae9f5dd020fac415dc798d5189697c2beb175f7b11160e1820ab3cc3c811 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:45:a4:e2:9c:d0\",\"name\":\"12d6ae9f5dd020f\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f49aa972-8af1-4459-a9f6-1b02a56e4aee\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:36.259388 3827 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:52:36.259429 3827 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 42.194µs W0329 02:52:40.624773 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0329 02:52:40.717246 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:40.717369 3827 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0329 02:52:40.740153 3827 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 22.841277ms I0329 02:52:40.740172 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 22.866619ms I0329 02:52:40.741192 3827 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 23.959892ms, libovsdb time 762.99µs I0329 02:52:41.131855 3827 cni.go:403] [openshift-monitoring/alertmanager-main-1 db6bb5248bc1619cac3cdddee2453a9a751b3b5979499a8a664ad500ca971d2f network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 db6bb5248bc1619cac3cdddee2453a9a751b3b5979499a8a664ad500ca971d2f network default NAD default] I0329 02:52:41.141191 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: db6bb5248bc1619, network: default, NAD default, SandboxID: "db6bb5248bc1619cac3cdddee2453a9a751b3b5979499a8a664ad500ca971d2f", PCI device ID: , UID: "8d098c9d-3007-4a0e-89f5-9a800f0099db", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0329 02:52:41.187246 3827 cni.go:424] [openshift-monitoring/alertmanager-main-1 db6bb5248bc1619cac3cdddee2453a9a751b3b5979499a8a664ad500ca971d2f network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 db6bb5248bc1619cac3cdddee2453a9a751b3b5979499a8a664ad500ca971d2f network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:59:39:5b:f6:f2\",\"name\":\"db6bb5248bc1619\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4d3a623-4126-4dc8-84c4-5ac554bde025\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:42.615888 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5] creating logical port openshift-monitoring_thanos-querier-c7fbdd87f-dhkf5 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:42.616015 3827 kube.go:256] Updating pod openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5 I0329 02:52:42.637743 3827 pod.go:62] [openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5] pod update took 21.752156ms I0329 02:52:42.637832 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5] addLogicalPort annotation time took 21.875305ms I0329 02:52:42.653679 3827 pods.go:271] [openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5] addLogicalPort took 37.807331ms, libovsdb time 15.105308ms I0329 02:52:43.005133 3827 cni.go:403] [openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5 e68ee646d91ee139fbf089cbf734f94274cf20ea4fd2487baf678d186612c4c8 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5 e68ee646d91ee139fbf089cbf734f94274cf20ea4fd2487baf678d186612c4c8 network default NAD default] I0329 02:52:43.013585 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-c7fbdd87f-dhkf5, hostIfaceName: e68ee646d91ee13, network: default, NAD default, SandboxID: "e68ee646d91ee139fbf089cbf734f94274cf20ea4fd2487baf678d186612c4c8", PCI device ID: , UID: "c240eb34-88ac-4b9d-8524-d4842d942cfd", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0329 02:52:43.054745 3827 cni.go:424] [openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5 e68ee646d91ee139fbf089cbf734f94274cf20ea4fd2487baf678d186612c4c8 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-c7fbdd87f-dhkf5 e68ee646d91ee139fbf089cbf734f94274cf20ea4fd2487baf678d186612c4c8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:7d:e2:9e:68:da\",\"name\":\"e68ee646d91ee13\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3497e7a-bc09-4d50-ab22-7a2c8e41c8c5\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 02:52:43.648136 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:43.648156 3827 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:43.648161 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0329 02:52:44.188432 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-krw98 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:44.188542 3827 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98 I0329 02:52:44.206657 3827 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98] pod update took 18.146142ms I0329 02:52:44.206681 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98] addLogicalPort annotation time took 18.176101ms I0329 02:52:44.207810 3827 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98] addLogicalPort took 19.399182ms, libovsdb time 822.235µs I0329 02:52:44.249805 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-7fd4674796-s7txz] creating logical port openshift-monitoring_metrics-server-7fd4674796-s7txz for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:44.249887 3827 kube.go:256] Updating pod openshift-monitoring/metrics-server-7fd4674796-s7txz I0329 02:52:44.276663 3827 pod.go:62] [openshift-monitoring/metrics-server-7fd4674796-s7txz] pod update took 26.796708ms I0329 02:52:44.276746 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-7fd4674796-s7txz] addLogicalPort annotation time took 26.882623ms I0329 02:52:44.278153 3827 pods.go:271] [openshift-monitoring/metrics-server-7fd4674796-s7txz] addLogicalPort took 28.360672ms, libovsdb time 966.646µs I0329 02:52:44.551356 3827 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98 07a9ea2859906318000d90921fc520e6686b4dc75974beb3b8154c39b236033e network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98 07a9ea2859906318000d90921fc520e6686b4dc75974beb3b8154c39b236033e network default NAD default] I0329 02:52:44.561106 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-krw98, hostIfaceName: 07a9ea285990631, network: default, NAD default, SandboxID: "07a9ea2859906318000d90921fc520e6686b4dc75974beb3b8154c39b236033e", PCI device ID: , UID: "b87006a1-8d80-4582-8970-e380addfadb8", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0329 02:52:44.607794 3827 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98 07a9ea2859906318000d90921fc520e6686b4dc75974beb3b8154c39b236033e network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-krw98 07a9ea2859906318000d90921fc520e6686b4dc75974beb3b8154c39b236033e network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:17:12:f3:fb:d5\",\"name\":\"07a9ea285990631\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be35759e-e196-44d8-9108-93ff6893e9a2\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:52:44.611509 3827 cni.go:403] [openshift-monitoring/metrics-server-7fd4674796-s7txz 432b3b3895fb8599dfe68a3c30b9fcfd5bf8836f6befdcb818da7221a777166e network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-7fd4674796-s7txz 432b3b3895fb8599dfe68a3c30b9fcfd5bf8836f6befdcb818da7221a777166e network default NAD default] I0329 02:52:44.619261 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-7fd4674796-s7txz, hostIfaceName: 432b3b3895fb859, network: default, NAD default, SandboxID: "432b3b3895fb8599dfe68a3c30b9fcfd5bf8836f6befdcb818da7221a777166e", PCI device ID: , UID: "716df424-c3f0-4300-8009-499f4af4d2ed", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0329 02:52:44.661261 3827 cni.go:424] [openshift-monitoring/metrics-server-7fd4674796-s7txz 432b3b3895fb8599dfe68a3c30b9fcfd5bf8836f6befdcb818da7221a777166e network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-7fd4674796-s7txz 432b3b3895fb8599dfe68a3c30b9fcfd5bf8836f6befdcb818da7221a777166e network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:61:12:7b:53:cb\",\"name\":\"432b3b3895fb859\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c52abf50-643e-48a7-8e8d-7d36deda09d7\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 02:52:45.108604 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:45.108626 3827 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:45.108630 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0329 02:52:45.916603 3827 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-37.ec2.internal I0329 02:52:45.916795 3827 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0329 02:52:45.949559 3827 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 32.816648ms I0329 02:52:45.949584 3827 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 32.848813ms I0329 02:52:45.950951 3827 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 34.361427ms, libovsdb time 985.382µs I0329 02:52:47.144849 3827 cni.go:403] [openshift-monitoring/prometheus-k8s-1 df38ff64562bc15d1df7439b4171bddef29b1945739c0e11d5b5895f0c8328a1 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 df38ff64562bc15d1df7439b4171bddef29b1945739c0e11d5b5895f0c8328a1 network default NAD default] I0329 02:52:47.153557 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: df38ff64562bc15, network: default, NAD default, SandboxID: "df38ff64562bc15d1df7439b4171bddef29b1945739c0e11d5b5895f0c8328a1", PCI device ID: , UID: "f7d24132-b13a-4f02-b920-1036f7e5efb8", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0329 02:52:47.200375 3827 cni.go:424] [openshift-monitoring/prometheus-k8s-1 df38ff64562bc15d1df7439b4171bddef29b1945739c0e11d5b5895f0c8328a1 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 df38ff64562bc15d1df7439b4171bddef29b1945739c0e11d5b5895f0c8328a1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:df:43:d4:4d:37\",\"name\":\"df38ff64562bc15\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01d35fef-94a1-47f9-a95a-9b78e04acb35\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 02:52:47.671167 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.671194 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.671203 3827 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.671209 3827 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.694862 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0329 02:52:47.787546 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:47.787568 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:51.149883 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0329 02:52:51.232440 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:51.232455 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:51.434048 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434074 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434082 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434089 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434108 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434115 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434121 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.434130 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:51.725883 3827 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-d477cd6dd-f9sdw IP address from the namespace address-set, err: pod openshift-console/console-d477cd6dd-f9sdw: no pod IPs found W0329 02:52:52.598301 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:53.692875 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:53.692898 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:53.692904 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:53.692909 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:53:01.052529 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0329 02:53:08.683721 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0329 02:53:09.384125 3827 cni.go:403] [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default] DEL starting CNI request [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default] I0329 02:53:09.458758 3827 cni.go:424] [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default] DEL finished CNI request [openshift-console/console-7677b9b887-frl82 a126cdfb37dda287604f4a457eb38ca193364f5fae59a2d3bf41f7b4558c71c0 network default NAD default], result "{}", err I0329 02:53:09.525315 3827 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-frl82 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:09.525354 3827 pods.go:176] Deleting pod: openshift-console/console-7677b9b887-frl82 I0329 02:53:09.526651 3827 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7677b9b887-frl82, ips: 10.129.0.12 I0329 02:53:09.526684 3827 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-frl82 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:09.536129 3827 base_network_controller_pods.go:487] [default/openshift-console/console-86449784f9-82tnj] creating logical port openshift-console_console-86449784f9-82tnj for pod on switch ip-10-0-1-37.ec2.internal I0329 02:53:09.536268 3827 kube.go:256] Updating pod openshift-console/console-86449784f9-82tnj I0329 02:53:09.552103 3827 pod.go:62] [openshift-console/console-86449784f9-82tnj] pod update took 15.877361ms I0329 02:53:09.552126 3827 base_network_controller_pods.go:937] [default/openshift-console/console-86449784f9-82tnj] addLogicalPort annotation time took 15.905086ms I0329 02:53:09.553141 3827 pods.go:271] [openshift-console/console-86449784f9-82tnj] addLogicalPort took 17.030915ms, libovsdb time 697.029µs I0329 02:53:09.763363 3827 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-frl82 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:09.763387 3827 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-frl82 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:09.771139 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7677b9b887-frl82 I0329 02:53:09.771157 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7677b9b887-frl82 I0329 02:53:09.942210 3827 cni.go:403] [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default] ADD starting CNI request [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default] I0329 02:53:09.950990 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-86449784f9-82tnj, hostIfaceName: d12181e87e6b310, network: default, NAD default, SandboxID: "d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd", PCI device ID: , UID: "06583e93-7d28-42f8-be79-92f13c93558d", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0329 02:53:10.199726 3827 cni.go:424] [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default] ADD finished CNI request [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:2c:56:36:26:de\",\"name\":\"d12181e87e6b310\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9237af92-195f-4f8e-9db7-3bb33437c825\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 02:53:11.168193 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:53:11.168213 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:53:11.168219 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:53:12.111355 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0329 02:53:18.281169 3827 obj_retry.go:464] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:18.281865 3827 obj_retry.go:464] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:18.435156 3827 obj_retry.go:464] Detected object openshift-console/console-d477cd6dd-plpzq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:18.435689 3827 obj_retry.go:464] Detected object openshift-console/console-d477cd6dd-plpzq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:19.148842 3827 obj_retry.go:458] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.148869 3827 obj_retry.go:458] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.155911 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-64fc4f6495-5pq2g I0329 02:53:19.155954 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-64fc4f6495-5pq2g I0329 02:53:19.262965 3827 obj_retry.go:458] Detected object openshift-console/console-d477cd6dd-plpzq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.262993 3827 obj_retry.go:458] Detected object openshift-console/console-d477cd6dd-plpzq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.271673 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-d477cd6dd-plpzq I0329 02:53:19.271747 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-d477cd6dd-plpzq W0329 02:53:20.838723 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0329 02:53:21.164281 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0329 02:53:27.317025 3827 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.317543 3827 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.317569 3827 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.317577 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0329 02:53:27.317591 3827 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.317599 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0329 02:53:28.345768 3827 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:28.345790 3827 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:28.345805 3827 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:28.345817 3827 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:30.325650 3827 obj_retry.go:464] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:30.326163 3827 obj_retry.go:464] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:31.212384 3827 obj_retry.go:458] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:31.212411 3827 obj_retry.go:458] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:31.223339 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7bf7699dd9-knnqb I0329 02:53:31.223364 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7bf7699dd9-knnqb I0329 02:53:32.152303 3827 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0329 02:53:32.163108 3827 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0329 02:53:45.379020 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:53:45.379068 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:53:45.379083 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:53:45.379089 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0329 02:53:46.177436 3827 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-84h62 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:46.178001 3827 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-84h62 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:46.523393 3827 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0329 02:53:46.578104 3827 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-84h62 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:46.578135 3827 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-84h62 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:46.586671 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7677b9b887-84h62 I0329 02:53:46.586698 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7677b9b887-84h62 W0329 02:53:46.949068 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:53:46.949092 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0329 02:54:33.933842 3827 cni.go:403] [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default] DEL starting CNI request [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default] I0329 02:54:34.017164 3827 cni.go:424] [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default] DEL finished CNI request [openshift-console/console-86449784f9-82tnj d12181e87e6b3102d1486ed5d8265fb94c82e5a01ebacc60f49ea4e73b1717fd network default NAD default], result "{}", err I0329 02:54:34.071818 3827 obj_retry.go:464] Detected object openshift-console/console-86449784f9-82tnj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:34.071867 3827 pods.go:176] Deleting pod: openshift-console/console-86449784f9-82tnj I0329 02:54:34.073001 3827 pods.go:236] Attempting to release IPs for pod: openshift-console/console-86449784f9-82tnj, ips: 10.129.0.21 I0329 02:54:34.073050 3827 obj_retry.go:464] Detected object openshift-console/console-86449784f9-82tnj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:34.082069 3827 base_network_controller_pods.go:487] [default/openshift-console/console-f66787c65-jhkvq] creating logical port openshift-console_console-f66787c65-jhkvq for pod on switch ip-10-0-1-37.ec2.internal I0329 02:54:34.082186 3827 kube.go:256] Updating pod openshift-console/console-f66787c65-jhkvq I0329 02:54:34.099705 3827 pod.go:62] [openshift-console/console-f66787c65-jhkvq] pod update took 17.543455ms I0329 02:54:34.099733 3827 base_network_controller_pods.go:937] [default/openshift-console/console-f66787c65-jhkvq] addLogicalPort annotation time took 17.573341ms I0329 02:54:34.100778 3827 pods.go:271] [openshift-console/console-f66787c65-jhkvq] addLogicalPort took 18.724811ms, libovsdb time 752.029µs I0329 02:54:34.391211 3827 obj_retry.go:458] Detected object openshift-console/console-86449784f9-82tnj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:34.391244 3827 obj_retry.go:458] Detected object openshift-console/console-86449784f9-82tnj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:34.398251 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86449784f9-82tnj I0329 02:54:34.398285 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86449784f9-82tnj I0329 02:54:34.441919 3827 cni.go:403] [openshift-console/console-f66787c65-jhkvq 1eecd05df3dc3dfe2ffefdbf4834ee6f7841d04a3f13779440e1c4b5d8f49f35 network default NAD default] ADD starting CNI request [openshift-console/console-f66787c65-jhkvq 1eecd05df3dc3dfe2ffefdbf4834ee6f7841d04a3f13779440e1c4b5d8f49f35 network default NAD default] I0329 02:54:34.450207 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-f66787c65-jhkvq, hostIfaceName: 1eecd05df3dc3df, network: default, NAD default, SandboxID: "1eecd05df3dc3dfe2ffefdbf4834ee6f7841d04a3f13779440e1c4b5d8f49f35", PCI device ID: , UID: "d0975685-89cb-450c-bcd0-2d37de69316c", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0329 02:54:34.491384 3827 cni.go:424] [openshift-console/console-f66787c65-jhkvq 1eecd05df3dc3dfe2ffefdbf4834ee6f7841d04a3f13779440e1c4b5d8f49f35 network default NAD default] ADD finished CNI request [openshift-console/console-f66787c65-jhkvq 1eecd05df3dc3dfe2ffefdbf4834ee6f7841d04a3f13779440e1c4b5d8f49f35 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:8c:a4:5c:69:7f\",\"name\":\"1eecd05df3dc3df\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b721312b-53b5-429f-8c49-ddf1bfe8d037\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 02:54:40.689468 3827 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:40.690097 3827 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:41.716420 3827 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:41.716452 3827 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:44.765149 3827 obj_retry.go:464] Detected object openshift-console/console-86449784f9-k4vn9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:44.765687 3827 obj_retry.go:464] Detected object openshift-console/console-86449784f9-k4vn9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0329 02:54:45.132926 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0329 02:54:45.568879 3827 obj_retry.go:458] Detected object openshift-console/console-86449784f9-k4vn9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:45.568911 3827 obj_retry.go:458] Detected object openshift-console/console-86449784f9-k4vn9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:45.575798 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86449784f9-k4vn9 I0329 02:54:45.575824 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86449784f9-k4vn9 I0329 02:57:21.595014 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=69 I0329 02:57:40.677681 3827 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0329 02:57:43.718026 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0329 02:58:02.717340 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0329 02:58:22.711850 3827 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 W0329 02:58:36.630285 3827 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:58:47.679567 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0329 02:58:47.679640 3827 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-sm2f7 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-sm2f7: no pod IPs found I0329 02:59:05.682850 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0329 02:59:38.604594 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=516 I0329 02:59:42.592389 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=110 I0329 03:00:04.128907 3827 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:00:04.129483 3827 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:00:05.024490 3827 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:00:05.024533 3827 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:00:28.706104 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0329 03:00:41.689280 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0329 03:00:51.700385 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0329 03:01:01.590869 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0329 03:01:02.864793 3827 namespace.go:98] [tsf] adding namespace I0329 03:01:02.865648 3827 namespace.go:102] [tsf] adding namespace took 831.661µs I0329 03:01:02.889000 3827 namespace.go:142] [tsf] updating namespace I0329 03:01:02.902295 3827 namespace.go:142] [tsf] updating namespace I0329 03:01:04.593374 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=81 I0329 03:01:04.695004 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0329 03:01:06.596141 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=20 I0329 03:01:06.687748 3827 namespace.go:98] [rhtpa-operator] adding namespace I0329 03:01:06.688565 3827 namespace.go:102] [rhtpa-operator] adding namespace took 801.389µs I0329 03:01:06.690573 3827 namespace.go:98] [tsf-tpa] adding namespace I0329 03:01:06.691165 3827 namespace.go:98] [konflux-operator] adding namespace I0329 03:01:06.691234 3827 namespace.go:102] [tsf-tpa] adding namespace took 644.556µs I0329 03:01:06.691854 3827 namespace.go:102] [konflux-operator] adding namespace took 674.344µs I0329 03:01:06.694932 3827 namespace.go:98] [tsf-tas] adding namespace I0329 03:01:06.695086 3827 namespace.go:98] [tsf-keycloak] adding namespace I0329 03:01:06.695571 3827 namespace.go:102] [tsf-tas] adding namespace took 622.868µs I0329 03:01:06.696112 3827 namespace.go:102] [tsf-keycloak] adding namespace took 1.011509ms I0329 03:01:06.696143 3827 namespace.go:98] [konflux-ui] adding namespace I0329 03:01:06.696583 3827 namespace.go:102] [konflux-ui] adding namespace took 430.24µs I0329 03:01:06.701085 3827 namespace.go:98] [rhbk-operator] adding namespace I0329 03:01:06.701134 3827 namespace.go:98] [cert-manager-operator] adding namespace I0329 03:01:06.701649 3827 namespace.go:102] [rhbk-operator] adding namespace took 551.315µs I0329 03:01:06.702201 3827 namespace.go:102] [cert-manager-operator] adding namespace took 1.052513ms I0329 03:01:06.725260 3827 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:01:06.734885 3827 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:01:06.757950 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:06.758877 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:06.763156 3827 namespace.go:142] [tsf-tas] updating namespace I0329 03:01:06.768582 3827 namespace.go:142] [konflux-operator] updating namespace I0329 03:01:06.776539 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:06.777615 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:01:06.781024 3827 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:01:06.791952 3827 namespace.go:142] [rhbk-operator] updating namespace I0329 03:01:06.811277 3827 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:01:06.837203 3827 namespace.go:142] [tsf-tas] updating namespace I0329 03:01:06.854300 3827 namespace.go:142] [tsf-tas] updating namespace I0329 03:01:06.867474 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:06.894103 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:06.903369 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:06.907237 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:01:06.920120 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:01:06.939694 3827 namespace.go:142] [rhbk-operator] updating namespace I0329 03:01:06.965929 3827 namespace.go:142] [rhbk-operator] updating namespace I0329 03:01:06.969023 3827 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:01:06.983822 3827 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:01:06.989080 3827 namespace.go:142] [konflux-operator] updating namespace I0329 03:01:07.002168 3827 namespace.go:142] [konflux-operator] updating namespace I0329 03:01:07.962911 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:07.962955 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:13.554217 3827 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.554744 3827 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.554774 3827 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.554784 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:13.554797 3827 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.554802 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:14.579857 3827 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:14.579883 3827 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:14.579897 3827 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:14.579909 3827 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:15.595195 3827 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.595746 3827 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.595780 3827 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.595791 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:15.595812 3827 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.595822 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:16.623704 3827 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:16.623731 3827 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:16.623744 3827 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:16.623756 3827 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:18.608608 3827 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.609228 3827 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.609256 3827 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.609265 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:18.609276 3827 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.609280 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:19.640799 3827 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:19.640823 3827 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:19.640837 3827 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:19.640850 3827 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:20.537635 3827 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.538203 3827 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.538227 3827 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.538235 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:20.538253 3827 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.538258 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:21.544080 3827 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.544542 3827 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.544566 3827 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.544577 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:21.544594 3827 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.544603 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:21.572266 3827 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.572287 3827 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.572299 3827 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.572308 3827 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.591003 3827 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.591471 3827 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.591492 3827 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.591499 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:21.591511 3827 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.591515 3827 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:22.572348 3827 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.572374 3827 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.572390 3827 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.572400 3827 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613149 3827 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613171 3827 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613184 3827 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613194 3827 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0329 03:01:27.463865 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0329 03:01:29.572174 3827 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-fnfb6 for pod on switch ip-10-0-1-37.ec2.internal I0329 03:01:29.572299 3827 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 I0329 03:01:29.612153 3827 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6] pod update took 39.878112ms I0329 03:01:29.612189 3827 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6] addLogicalPort annotation time took 39.918517ms I0329 03:01:29.613556 3827 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6] addLogicalPort took 41.403291ms, libovsdb time 937.051µs I0329 03:01:29.925286 3827 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default] I0329 03:01:29.933900 3827 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-fnfb6, hostIfaceName: 8639daff6745c83, network: default, NAD default, SandboxID: "8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d", PCI device ID: , UID: "2c06fb5b-7ead-4f0d-8644-6a6a1d32cf7f", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0329 03:01:29.979856 3827 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:51:6f:85:30:ff\",\"name\":\"8639daff6745c83\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1eb6af23-e247-4f93-8647-fc7272e02415\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 03:01:32.011335 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0329 03:01:32.043773 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0329 03:01:32.777441 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0329 03:01:34.002915 3827 namespace.go:98] [cert-manager] adding namespace I0329 03:01:34.003802 3827 namespace.go:102] [cert-manager] adding namespace took 859.766µs I0329 03:01:34.037124 3827 namespace.go:142] [cert-manager] updating namespace I0329 03:01:34.057914 3827 namespace.go:142] [cert-manager] updating namespace I0329 03:01:34.678573 3827 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0329 03:01:35.194542 3827 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-gqpdw for pod on switch ip-10-0-1-37.ec2.internal I0329 03:01:35.194639 3827 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw I0329 03:01:35.210735 3827 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw] pod update took 16.118925ms I0329 03:01:35.210762 3827 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw] addLogicalPort annotation time took 16.149591ms I0329 03:01:35.212205 3827 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw] addLogicalPort took 17.675756ms, libovsdb time 978.357µs I0329 03:01:35.565952 3827 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw fdbddb780724a5105c79e5f3f950956e810fec7d5ebd3af9b991b781fe95627e network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw fdbddb780724a5105c79e5f3f950956e810fec7d5ebd3af9b991b781fe95627e network default NAD default] I0329 03:01:35.574407 3827 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-gqpdw, hostIfaceName: fdbddb780724a51, network: default, NAD default, SandboxID: "fdbddb780724a5105c79e5f3f950956e810fec7d5ebd3af9b991b781fe95627e", PCI device ID: , UID: "75b75cfd-d298-434f-92e3-1f0c6811f594", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0329 03:01:35.621841 3827 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw fdbddb780724a5105c79e5f3f950956e810fec7d5ebd3af9b991b781fe95627e network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-gqpdw fdbddb780724a5105c79e5f3f950956e810fec7d5ebd3af9b991b781fe95627e network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:fa:47:3f:e9:73\",\"name\":\"fdbddb780724a51\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/61c83ff6-68d9-4105-945f-27d59df8f3b5\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:01:35.802644 3827 namespace.go:142] [openshift-operators] updating namespace I0329 03:01:35.826175 3827 namespace.go:142] [openshift-operators] updating namespace W0329 03:01:39.890999 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0329 03:01:39.891019 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0329 03:01:43.691960 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0329 03:01:45.729478 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0329 03:01:45.729503 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0329 03:01:46.308686 3827 namespace.go:98] [trusted-artifact-signer] adding namespace I0329 03:01:46.309556 3827 namespace.go:102] [trusted-artifact-signer] adding namespace took 847.169µs I0329 03:01:46.350115 3827 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:01:46.350228 3827 namespace.go:142] [trusted-artifact-signer] updating namespace W0329 03:01:46.655440 3827 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0329 03:01:49.351025 3827 namespace.go:142] [openshift-operators] updating namespace I0329 03:01:49.739901 3827 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-t9c2c for pod on switch ip-10-0-1-37.ec2.internal I0329 03:01:49.740017 3827 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c I0329 03:01:49.771199 3827 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c] pod update took 31.202894ms I0329 03:01:49.771227 3827 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c] addLogicalPort annotation time took 31.237699ms I0329 03:01:49.772363 3827 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c] addLogicalPort took 32.480802ms, libovsdb time 791.767µs I0329 03:01:50.098690 3827 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c ecc7aeacb053817e826c9437254e1141025a5e5b1bf2064de1dfaa40b888830f network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c ecc7aeacb053817e826c9437254e1141025a5e5b1bf2064de1dfaa40b888830f network default NAD default] I0329 03:01:50.107429 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-t9c2c, hostIfaceName: ecc7aeacb053817, network: default, NAD default, SandboxID: "ecc7aeacb053817e826c9437254e1141025a5e5b1bf2064de1dfaa40b888830f", PCI device ID: , UID: "0200e75b-bcf7-4ed2-9a25-8f1d41238d35", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0329 03:01:50.153702 3827 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c ecc7aeacb053817e826c9437254e1141025a5e5b1bf2064de1dfaa40b888830f network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-t9c2c ecc7aeacb053817e826c9437254e1141025a5e5b1bf2064de1dfaa40b888830f network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:ad:75:d3:7b:96\",\"name\":\"ecc7aeacb053817\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40173aa2-d43f-4abc-b1d0-b854f3ab4d97\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:01:54.794448 3827 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-l95md] creating logical port cert-manager_cert-manager-59ccb4bb6b-l95md for pod on switch ip-10-0-1-37.ec2.internal I0329 03:01:54.794535 3827 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-l95md I0329 03:01:54.820442 3827 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-l95md] pod update took 25.925391ms I0329 03:01:54.820475 3827 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-l95md] addLogicalPort annotation time took 25.96024ms I0329 03:01:54.821847 3827 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-l95md] addLogicalPort took 27.417744ms, libovsdb time 920.826µs I0329 03:01:55.152096 3827 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-l95md 5fba9aae1c6616a59ec54e0ed46f70e08e007b97ba3465368d6dd60c53ccacd5 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-l95md 5fba9aae1c6616a59ec54e0ed46f70e08e007b97ba3465368d6dd60c53ccacd5 network default NAD default] I0329 03:01:55.161594 3827 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-l95md, hostIfaceName: 5fba9aae1c6616a, network: default, NAD default, SandboxID: "5fba9aae1c6616a59ec54e0ed46f70e08e007b97ba3465368d6dd60c53ccacd5", PCI device ID: , UID: "8feb767b-303a-4fad-b3d4-3ae5c4e75bef", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0329 03:01:55.206302 3827 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-l95md 5fba9aae1c6616a59ec54e0ed46f70e08e007b97ba3465368d6dd60c53ccacd5 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-l95md 5fba9aae1c6616a59ec54e0ed46f70e08e007b97ba3465368d6dd60c53ccacd5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:c2:0d:c3:f7:c3\",\"name\":\"5fba9aae1c6616a\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9e097e6a-ef0a-48c8-ada4-5a7a762eef6d\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:02:04.167787 3827 namespace.go:98] [openshift-pipelines] adding namespace I0329 03:02:04.168651 3827 namespace.go:102] [openshift-pipelines] adding namespace took 842.123µs I0329 03:02:04.193100 3827 namespace.go:142] [openshift-pipelines] updating namespace I0329 03:02:04.882855 3827 namespace.go:142] [cert-manager] updating namespace I0329 03:02:05.081202 3827 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:02:05.484068 3827 namespace.go:142] [konflux-operator] updating namespace I0329 03:02:05.684462 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:02:05.887160 3827 namespace.go:142] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] updating namespace I0329 03:02:06.289816 3827 namespace.go:142] [rhbk-operator] updating namespace I0329 03:02:06.488747 3827 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:02:06.688051 3827 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:02:06.886925 3827 namespace.go:142] [tsf] updating namespace I0329 03:02:07.089674 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:02:07.290988 3827 namespace.go:142] [tsf-tas] updating namespace I0329 03:02:07.490285 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:02:07.629208 3827 namespace.go:142] [cert-manager] updating namespace I0329 03:02:07.643751 3827 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:02:07.655310 3827 namespace.go:142] [default] updating namespace I0329 03:02:07.670143 3827 namespace.go:142] [konflux-operator] updating namespace I0329 03:02:07.682384 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:02:07.692666 3827 namespace.go:142] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] updating namespace I0329 03:02:07.704623 3827 namespace.go:142] [openshift] updating namespace I0329 03:02:07.717260 3827 namespace.go:142] [rhbk-operator] updating namespace I0329 03:02:07.730907 3827 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:02:07.740721 3827 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:02:07.751671 3827 namespace.go:142] [tsf] updating namespace I0329 03:02:07.761520 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:02:07.777984 3827 namespace.go:142] [tsf-tas] updating namespace I0329 03:02:07.792653 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:02:07.876446 3827 namespace.go:142] [cert-manager] updating namespace I0329 03:02:07.977128 3827 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:02:08.073224 3827 namespace.go:142] [default] updating namespace I0329 03:02:08.174212 3827 namespace.go:142] [konflux-operator] updating namespace I0329 03:02:08.278254 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:02:08.372902 3827 namespace.go:142] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] updating namespace I0329 03:02:08.479502 3827 namespace.go:142] [openshift] updating namespace I0329 03:02:08.573830 3827 namespace.go:142] [rhbk-operator] updating namespace I0329 03:02:08.672146 3827 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:02:08.772702 3827 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:02:08.878286 3827 namespace.go:142] [tsf] updating namespace I0329 03:02:08.983363 3827 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:02:09.080265 3827 namespace.go:142] [tsf-tas] updating namespace I0329 03:02:09.190235 3827 namespace.go:142] [tsf-tpa] updating namespace I0329 03:02:09.603928 3827 namespace.go:142] [openshift-pipelines] updating namespace I0329 03:02:17.482215 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-gg9xf for pod on switch ip-10-0-1-37.ec2.internal I0329 03:02:17.482317 3827 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf I0329 03:02:17.504888 3827 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf] pod update took 22.598289ms I0329 03:02:17.504917 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf] addLogicalPort annotation time took 22.62885ms I0329 03:02:17.506079 3827 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf] addLogicalPort took 23.87678ms, libovsdb time 804.155µs I0329 03:02:17.760003 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-9ssdf for pod on switch ip-10-0-1-37.ec2.internal I0329 03:02:17.760104 3827 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf I0329 03:02:17.777179 3827 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf] pod update took 17.097622ms I0329 03:02:17.777201 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf] addLogicalPort annotation time took 17.128291ms I0329 03:02:17.778299 3827 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf] addLogicalPort took 18.311225ms, libovsdb time 786.706µs I0329 03:02:17.839006 3827 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf cd64d710c22109ed5cd3aaf1bdc1ac54c75c0003c6aa5e2be6332da1e58ea626 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf cd64d710c22109ed5cd3aaf1bdc1ac54c75c0003c6aa5e2be6332da1e58ea626 network default NAD default] I0329 03:02:17.848410 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-gg9xf, hostIfaceName: cd64d710c22109e, network: default, NAD default, SandboxID: "cd64d710c22109ed5cd3aaf1bdc1ac54c75c0003c6aa5e2be6332da1e58ea626", PCI device ID: , UID: "a2856585-3c9f-475d-8d14-762d93e23ebd", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0329 03:02:17.885702 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-2tzs5 for pod on switch ip-10-0-1-37.ec2.internal I0329 03:02:17.885815 3827 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5 I0329 03:02:17.895892 3827 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf cd64d710c22109ed5cd3aaf1bdc1ac54c75c0003c6aa5e2be6332da1e58ea626 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-gg9xf cd64d710c22109ed5cd3aaf1bdc1ac54c75c0003c6aa5e2be6332da1e58ea626 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:e6:78:04:dc:6d\",\"name\":\"cd64d710c22109e\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8bfd9ee6-fef0-451d-bb59-c786ad3025e4\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:02:17.920880 3827 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5] pod update took 35.100211ms I0329 03:02:17.920905 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5] addLogicalPort annotation time took 35.127669ms I0329 03:02:17.922057 3827 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5] addLogicalPort took 36.374739ms, libovsdb time 720.056µs I0329 03:02:18.118732 3827 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf 6f127847225ceb5220ac6efcbfb54f68399060e01e7150ba89857d1c759cf0bf network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf 6f127847225ceb5220ac6efcbfb54f68399060e01e7150ba89857d1c759cf0bf network default NAD default] I0329 03:02:18.127647 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-9ssdf, hostIfaceName: 6f127847225ceb5, network: default, NAD default, SandboxID: "6f127847225ceb5220ac6efcbfb54f68399060e01e7150ba89857d1c759cf0bf", PCI device ID: , UID: "a18fc03c-8596-4867-b96e-22896e4306c3", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0329 03:02:18.171500 3827 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf 6f127847225ceb5220ac6efcbfb54f68399060e01e7150ba89857d1c759cf0bf network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-9ssdf 6f127847225ceb5220ac6efcbfb54f68399060e01e7150ba89857d1c759cf0bf network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:28:2b:b3:53:a6\",\"name\":\"6f127847225ceb5\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/161f64a1-6f54-4bb8-83cb-557192bfdf99\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:02:18.245007 3827 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5 c7f4ba82950bbdcad6e15b8ec7f1675bfef7b6baa6e340c7b17f4a34a58b184c network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5 c7f4ba82950bbdcad6e15b8ec7f1675bfef7b6baa6e340c7b17f4a34a58b184c network default NAD default] I0329 03:02:18.254862 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-2tzs5, hostIfaceName: c7f4ba82950bbdc, network: default, NAD default, SandboxID: "c7f4ba82950bbdcad6e15b8ec7f1675bfef7b6baa6e340c7b17f4a34a58b184c", PCI device ID: , UID: "00bbe14d-4c55-495c-a1de-edf837612a6f", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0329 03:02:18.509216 3827 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5 c7f4ba82950bbdcad6e15b8ec7f1675bfef7b6baa6e340c7b17f4a34a58b184c network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-2tzs5 c7f4ba82950bbdcad6e15b8ec7f1675bfef7b6baa6e340c7b17f4a34a58b184c network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:15:70:20:4b:a7\",\"name\":\"c7f4ba82950bbdc\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2417420a-00ff-47fb-b027-0957c24c1f1d\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:02:19.455836 3827 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:19.456445 3827 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:19.478919 3827 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:19.478949 3827 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:19.486602 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0329 03:02:19.486719 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0329 03:02:20.894293 3827 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:20.894312 3827 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:20.894317 3827 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:21.331483 3827 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0329 03:02:21.331508 3827 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0329 03:02:21.331520 3827 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0329 03:02:21.923410 3827 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:21.923432 3827 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:21.923438 3827 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:21.923442 3827 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:31.389732 3827 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0329 03:02:31.415633 3827 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0329 03:02:31.902533 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:31.902553 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:31.902558 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:33.913236 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:33.913255 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:33.913261 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:33.913265 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:46.935517 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0329 03:02:54.646916 3827 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:54.647457 3827 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:54.676146 3827 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:54.676167 3827 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:54.682025 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0329 03:02:54.682058 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0329 03:02:59.852772 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-9kv2t for pod on switch ip-10-0-1-37.ec2.internal I0329 03:02:59.852880 3827 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t I0329 03:02:59.867843 3827 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t] pod update took 14.993216ms I0329 03:02:59.867868 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t] addLogicalPort annotation time took 15.024349ms I0329 03:02:59.869273 3827 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t] addLogicalPort took 16.521718ms, libovsdb time 1.016018ms I0329 03:03:00.220459 3827 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t a7a9c2433c0c663ad1d795b9d612ec1f8d9b32c396b5abe49104c83bb98a1cc3 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t a7a9c2433c0c663ad1d795b9d612ec1f8d9b32c396b5abe49104c83bb98a1cc3 network default NAD default] I0329 03:03:00.229111 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-9kv2t, hostIfaceName: a7a9c2433c0c663, network: default, NAD default, SandboxID: "a7a9c2433c0c663ad1d795b9d612ec1f8d9b32c396b5abe49104c83bb98a1cc3", PCI device ID: , UID: "0dd42e37-6e38-48b3-aac6-e641905d1790", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0329 03:03:00.274902 3827 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t a7a9c2433c0c663ad1d795b9d612ec1f8d9b32c396b5abe49104c83bb98a1cc3 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-9kv2t a7a9c2433c0c663ad1d795b9d612ec1f8d9b32c396b5abe49104c83bb98a1cc3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:ed:ab:bd:a2:98\",\"name\":\"a7a9c2433c0c663\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24aad783-0ebf-4265-89bf-f6e6bbda7af3\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:03:04.218503 3827 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0329 03:03:04.564121 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:04.564148 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:04.564212 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0329 03:03:24.539175 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-j7t65 for pod on switch ip-10-0-1-37.ec2.internal I0329 03:03:24.539282 3827 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65 I0329 03:03:24.556657 3827 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65] pod update took 17.400293ms I0329 03:03:24.556680 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65] addLogicalPort annotation time took 17.428128ms I0329 03:03:24.557849 3827 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65] addLogicalPort took 18.693474ms, libovsdb time 779.4µs I0329 03:03:24.743542 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-gbmhf for pod on switch ip-10-0-1-37.ec2.internal I0329 03:03:24.743664 3827 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf I0329 03:03:24.760060 3827 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf] pod update took 16.39281ms I0329 03:03:24.760087 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf] addLogicalPort annotation time took 16.448664ms I0329 03:03:24.761291 3827 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf] addLogicalPort took 17.765992ms, libovsdb time 851.552µs I0329 03:03:24.897250 3827 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65 ace2fe38604c93537911145f54669e8cc3da211154717ba76aabfd6f07b99a60 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65 ace2fe38604c93537911145f54669e8cc3da211154717ba76aabfd6f07b99a60 network default NAD default] I0329 03:03:24.905577 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-j7t65, hostIfaceName: ace2fe38604c935, network: default, NAD default, SandboxID: "ace2fe38604c93537911145f54669e8cc3da211154717ba76aabfd6f07b99a60", PCI device ID: , UID: "7b0cdc5a-fbc2-4b03-a380-a4261c394400", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0329 03:03:24.925700 3827 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:24.926229 3827 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:24.951176 3827 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65 ace2fe38604c93537911145f54669e8cc3da211154717ba76aabfd6f07b99a60 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-j7t65 ace2fe38604c93537911145f54669e8cc3da211154717ba76aabfd6f07b99a60 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:68:d2:f3:8f:d3\",\"name\":\"ace2fe38604c935\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6d0c2ab9-8333-47d1-b66f-4b7fd6b0a5e5\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:03:25.103810 3827 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf 3e2b114f0d5684af3ebfbc12a90d036fb703edf16707eee26378815f11a66899 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf 3e2b114f0d5684af3ebfbc12a90d036fb703edf16707eee26378815f11a66899 network default NAD default] I0329 03:03:25.112158 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-gbmhf, hostIfaceName: 3e2b114f0d5684a, network: default, NAD default, SandboxID: "3e2b114f0d5684af3ebfbc12a90d036fb703edf16707eee26378815f11a66899", PCI device ID: , UID: "2f1fa89d-75ea-4d93-a41f-4f135b8b1084", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0329 03:03:25.156500 3827 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf 3e2b114f0d5684af3ebfbc12a90d036fb703edf16707eee26378815f11a66899 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-gbmhf 3e2b114f0d5684af3ebfbc12a90d036fb703edf16707eee26378815f11a66899 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:fa:bb:60:8a:59\",\"name\":\"3e2b114f0d5684a\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2217619d-7f00-4651-b56e-141ad60ea748\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:03:25.669263 3827 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:25.669288 3827 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:25.676528 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0329 03:03:25.676552 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0329 03:03:28.697672 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0329 03:03:28.697729 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:28.697749 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:30.250069 3827 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:30.250089 3827 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:30.250093 3827 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:35.255633 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:35.255658 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:35.255664 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0329 03:03:41.330106 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-mkqgf] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-mkqgf for pod on switch ip-10-0-1-37.ec2.internal I0329 03:03:41.330196 3827 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-mkqgf I0329 03:03:41.344658 3827 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-mkqgf] pod update took 14.482344ms I0329 03:03:41.344689 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-mkqgf] addLogicalPort annotation time took 14.516716ms I0329 03:03:41.346176 3827 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-mkqgf] addLogicalPort took 16.089122ms, libovsdb time 957.72µs I0329 03:03:41.687515 3827 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-mkqgf fafa21e343d093fbe9a47c67a0066d77b095a33dfdbd20c504d078e21d6e64ef network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-mkqgf fafa21e343d093fbe9a47c67a0066d77b095a33dfdbd20c504d078e21d6e64ef network default NAD default] I0329 03:03:41.696584 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-mkqgf, hostIfaceName: fafa21e343d093f, network: default, NAD default, SandboxID: "fafa21e343d093fbe9a47c67a0066d77b095a33dfdbd20c504d078e21d6e64ef", PCI device ID: , UID: "80dd6087-655e-4da0-a47c-0e6f4da9d250", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0329 03:03:41.741636 3827 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-mkqgf fafa21e343d093fbe9a47c67a0066d77b095a33dfdbd20c504d078e21d6e64ef network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-mkqgf fafa21e343d093fbe9a47c67a0066d77b095a33dfdbd20c504d078e21d6e64ef network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:79:70:88:20:6c\",\"name\":\"fafa21e343d093f\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/660a3783-2d04-488b-9a14-53a51d12a241\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:03:42.597706 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=46 I0329 03:03:51.008767 3827 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:51.009254 3827 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:51.051568 3827 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:51.051590 3827 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:51.061077 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0329 03:03:51.061113 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0329 03:03:52.787143 3827 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-25s7b-vh7x4] creating logical port tsf-tas_segment-backup-installation-25s7b-vh7x4 for pod on switch ip-10-0-1-37.ec2.internal I0329 03:03:52.787234 3827 kube.go:256] Updating pod tsf-tas/segment-backup-installation-25s7b-vh7x4 I0329 03:03:52.802496 3827 pod.go:62] [tsf-tas/segment-backup-installation-25s7b-vh7x4] pod update took 15.283484ms I0329 03:03:52.802518 3827 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-25s7b-vh7x4] addLogicalPort annotation time took 15.311122ms I0329 03:03:52.803744 3827 pods.go:271] [tsf-tas/segment-backup-installation-25s7b-vh7x4] addLogicalPort took 16.616041ms, libovsdb time 838.774µs I0329 03:03:53.145072 3827 cni.go:403] [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default] I0329 03:03:53.154983 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-25s7b-vh7x4, hostIfaceName: cb291aff9b4654c, network: default, NAD default, SandboxID: "cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7", PCI device ID: , UID: "19f4fc73-ce96-4489-a5d5-70ab417968d9", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0329 03:03:53.202254 3827 cni.go:424] [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:6f:2d:93:4a:6b\",\"name\":\"cb291aff9b4654c\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a74dbbf1-1a15-49dc-860c-1c2658326916\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 03:03:56.363067 3827 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0329 03:03:56.363092 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0329 03:03:57.000347 3827 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-6d4c56d4f8-s7ltf] creating logical port tsf-tas_trillian-db-6d4c56d4f8-s7ltf for pod on switch ip-10-0-1-37.ec2.internal I0329 03:03:57.000473 3827 kube.go:256] Updating pod tsf-tas/trillian-db-6d4c56d4f8-s7ltf I0329 03:03:57.014578 3827 pod.go:62] [tsf-tas/trillian-db-6d4c56d4f8-s7ltf] pod update took 14.140081ms I0329 03:03:57.014605 3827 base_network_controller_pods.go:937] [default/tsf-tas/trillian-db-6d4c56d4f8-s7ltf] addLogicalPort annotation time took 14.174295ms I0329 03:03:57.015992 3827 pods.go:271] [tsf-tas/trillian-db-6d4c56d4f8-s7ltf] addLogicalPort took 15.663778ms, libovsdb time 947.903µs I0329 03:04:01.619790 3827 cni.go:403] [tsf-tas/trillian-db-6d4c56d4f8-s7ltf 8852d34eacde20913b0bc97a9e9a8fca17792f34a90bea5359c1f94c3f1e27bf network default NAD default] ADD starting CNI request [tsf-tas/trillian-db-6d4c56d4f8-s7ltf 8852d34eacde20913b0bc97a9e9a8fca17792f34a90bea5359c1f94c3f1e27bf network default NAD default] I0329 03:04:01.628957 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-6d4c56d4f8-s7ltf, hostIfaceName: 8852d34eacde209, network: default, NAD default, SandboxID: "8852d34eacde20913b0bc97a9e9a8fca17792f34a90bea5359c1f94c3f1e27bf", PCI device ID: , UID: "542729c1-d739-49dc-beb4-5278ca1b888e", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0329 03:04:01.676967 3827 cni.go:424] [tsf-tas/trillian-db-6d4c56d4f8-s7ltf 8852d34eacde20913b0bc97a9e9a8fca17792f34a90bea5359c1f94c3f1e27bf network default NAD default] ADD finished CNI request [tsf-tas/trillian-db-6d4c56d4f8-s7ltf 8852d34eacde20913b0bc97a9e9a8fca17792f34a90bea5359c1f94c3f1e27bf network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:24:7f:a5:cc:ac\",\"name\":\"8852d34eacde209\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6e364a6e-ec8c-4733-a13c-89a8a4aabef4\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:04:03.053364 3827 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-4mqwg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:03.053947 3827 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-4mqwg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:04.081159 3827 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-4mqwg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.081242 3827 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-4mqwg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.426974 3827 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:04.427602 3827 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:04.907011 3827 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.907052 3827 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.916192 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0329 03:04:04.916223 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0329 03:04:05.409459 3827 cni.go:403] [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default] I0329 03:04:05.502370 3827 cni.go:424] [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-25s7b-vh7x4 cb291aff9b4654c88b1945c47dc97813afe12cc99d6229b1a4499a80db75dde7 network default NAD default], result "{}", err I0329 03:04:06.629154 3827 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:06.629192 3827 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-25s7b-vh7x4 I0329 03:04:06.630649 3827 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-25s7b-vh7x4, ips: 10.129.0.34 I0329 03:04:06.630697 3827 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0329 03:04:06.931278 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:04:06.931297 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:04:06.931350 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0329 03:04:07.439827 3827 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0329 03:04:07.649302 3827 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:07.649332 3827 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:08.680171 3827 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0329 03:04:08.685153 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 W0329 03:04:10.453490 3827 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0329 03:04:10.481112 3827 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0329 03:04:12.179399 3827 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-pd2s2 for pod on switch ip-10-0-1-37.ec2.internal I0329 03:04:12.179497 3827 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2 I0329 03:04:12.197314 3827 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2] pod update took 17.838749ms I0329 03:04:12.197340 3827 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2] addLogicalPort annotation time took 17.869715ms I0329 03:04:12.198702 3827 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2] addLogicalPort took 19.317099ms, libovsdb time 830.738µs I0329 03:04:12.536626 3827 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2 a52a6b5cc79ef00b14cbdf67911f40d76ec89c446dd871ffa053c13c44539c3b network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2 a52a6b5cc79ef00b14cbdf67911f40d76ec89c446dd871ffa053c13c44539c3b network default NAD default] I0329 03:04:12.544968 3827 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-pd2s2, hostIfaceName: a52a6b5cc79ef00, network: default, NAD default, SandboxID: "a52a6b5cc79ef00b14cbdf67911f40d76ec89c446dd871ffa053c13c44539c3b", PCI device ID: , UID: "9ca31eab-da83-4353-8448-e1341a780275", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0329 03:04:12.592655 3827 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2 a52a6b5cc79ef00b14cbdf67911f40d76ec89c446dd871ffa053c13c44539c3b network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pd2s2 a52a6b5cc79ef00b14cbdf67911f40d76ec89c446dd871ffa053c13c44539c3b network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:a1:30:f6:ac:0d\",\"name\":\"a52a6b5cc79ef00\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fff12460-e2e7-4d85-98fa-71957afdfa4c\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 03:04:15.993855 3827 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0329 03:04:18.459371 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0329 03:04:41.240891 3827 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:41.241504 3827 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:41.262003 3827 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:41.262023 3827 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:41.269087 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0329 03:04:41.269112 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0329 03:04:49.295529 3827 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:49.296129 3827 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:50.178419 3827 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:50.178455 3827 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:50.298957 3827 obj_retry.go:464] Detected object tsf/patch-tekton-config-tzqlf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:50.299567 3827 obj_retry.go:464] Detected object tsf/patch-tekton-config-tzqlf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:51.322126 3827 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:51.322151 3827 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:51.428440 3827 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-j5v7l] creating logical port tsf_tsf-tekton-configuration-j5v7l for pod on switch ip-10-0-1-37.ec2.internal I0329 03:04:51.428530 3827 kube.go:256] Updating pod tsf/tsf-tekton-configuration-j5v7l I0329 03:04:51.441487 3827 pod.go:62] [tsf/tsf-tekton-configuration-j5v7l] pod update took 12.978433ms I0329 03:04:51.441510 3827 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-j5v7l] addLogicalPort annotation time took 13.006641ms I0329 03:04:51.442929 3827 pods.go:271] [tsf/tsf-tekton-configuration-j5v7l] addLogicalPort took 14.506318ms, libovsdb time 988.537µs I0329 03:04:51.790003 3827 cni.go:403] [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default] I0329 03:04:51.798405 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-j5v7l, hostIfaceName: 7d039aae68c6e67, network: default, NAD default, SandboxID: "7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215", PCI device ID: , UID: "831f1266-d529-4d84-a627-141e88bbad96", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0329 03:04:51.844671 3827 cni.go:424] [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:4a:f4:61:ab:9c\",\"name\":\"7d039aae68c6e67\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9daba65-ce4e-4453-91c7-8022def5f1d7\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 03:04:54.184204 3827 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0329 03:04:56.741805 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0329 03:04:57.893198 3827 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:57.893729 3827 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:58.672148 3827 cni.go:403] [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default] I0329 03:04:58.756720 3827 cni.go:424] [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-j5v7l 7d039aae68c6e677f447d9651d6e903833a0d0b5de97b60a1895d6aa0396c215 network default NAD default], result "{}", err I0329 03:04:58.791838 3827 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:58.791862 3827 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:58.806238 3827 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-j5v7l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:58.806273 3827 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-j5v7l I0329 03:04:58.807644 3827 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-j5v7l, ips: 10.129.0.37 I0329 03:04:58.807669 3827 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-j5v7l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:59.249798 3827 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-57979f4db-tb5lx] creating logical port tsf-tas_ctlog-57979f4db-tb5lx for pod on switch ip-10-0-1-37.ec2.internal I0329 03:04:59.249887 3827 kube.go:256] Updating pod tsf-tas/ctlog-57979f4db-tb5lx I0329 03:04:59.267148 3827 pod.go:62] [tsf-tas/ctlog-57979f4db-tb5lx] pod update took 17.282002ms I0329 03:04:59.267170 3827 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-57979f4db-tb5lx] addLogicalPort annotation time took 17.309581ms I0329 03:04:59.268361 3827 pods.go:271] [tsf-tas/ctlog-57979f4db-tb5lx] addLogicalPort took 18.583411ms, libovsdb time 791.229µs I0329 03:04:59.685490 3827 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-j5v7l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.685514 3827 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-j5v7l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.735937 3827 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.735958 3827 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.743463 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-tzqlf I0329 03:04:59.743492 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-tzqlf I0329 03:05:00.207188 3827 cni.go:403] [tsf-tas/ctlog-57979f4db-tb5lx cdb2395d963446053087054e59399271b17a9d09da1f4f295bb1e245feba39af network default NAD default] ADD starting CNI request [tsf-tas/ctlog-57979f4db-tb5lx cdb2395d963446053087054e59399271b17a9d09da1f4f295bb1e245feba39af network default NAD default] I0329 03:05:00.215846 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-57979f4db-tb5lx, hostIfaceName: cdb2395d9634460, network: default, NAD default, SandboxID: "cdb2395d963446053087054e59399271b17a9d09da1f4f295bb1e245feba39af", PCI device ID: , UID: "0657d245-bcd9-4324-9ff6-9c006a0ca48d", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0329 03:05:00.467226 3827 cni.go:424] [tsf-tas/ctlog-57979f4db-tb5lx cdb2395d963446053087054e59399271b17a9d09da1f4f295bb1e245feba39af network default NAD default] ADD finished CNI request [tsf-tas/ctlog-57979f4db-tb5lx cdb2395d963446053087054e59399271b17a9d09da1f4f295bb1e245feba39af network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:ad:b1:01:fe:94\",\"name\":\"cdb2395d9634460\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a020317-f988-42e2-bb64-f80d2f0eca4e\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 03:05:03.696654 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0329 03:05:05.240235 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0329 03:05:06.714430 3827 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0329 03:05:10.980985 3827 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:10.981544 3827 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:10.999992 3827 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:11.000016 3827 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:11.008287 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0329 03:05:11.008312 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines W0329 03:05:14.707405 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0329 03:05:15.434427 3827 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:15.434965 3827 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:15.456156 3827 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:15.456181 3827 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:15.464518 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0329 03:05:15.464544 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0329 03:05:16.752102 3827 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-w97nj] creating logical port tsf-tpa_create-db-w97nj for pod on switch ip-10-0-1-37.ec2.internal I0329 03:05:16.752246 3827 kube.go:256] Updating pod tsf-tpa/create-db-w97nj I0329 03:05:16.766329 3827 pod.go:62] [tsf-tpa/create-db-w97nj] pod update took 14.110915ms I0329 03:05:16.766352 3827 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-w97nj] addLogicalPort annotation time took 14.140782ms I0329 03:05:16.767687 3827 pods.go:271] [tsf-tpa/create-db-w97nj] addLogicalPort took 15.602854ms, libovsdb time 901.546µs I0329 03:05:17.114742 3827 cni.go:403] [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default] ADD starting CNI request [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default] I0329 03:05:17.124027 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-w97nj, hostIfaceName: b422fa27d2e9b3a, network: default, NAD default, SandboxID: "b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7", PCI device ID: , UID: "35929f5c-dacb-4f5e-b742-086b8a153fd7", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0329 03:05:17.169668 3827 cni.go:424] [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default] ADD finished CNI request [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:5b:6e:0b:65:c4\",\"name\":\"b422fa27d2e9b3a\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c6731163-1088-4432-80d0-cd3edf6e08e1\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:05:22.802460 3827 cni.go:403] [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default] DEL starting CNI request [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default] I0329 03:05:22.892106 3827 cni.go:424] [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default] DEL finished CNI request [tsf-tpa/create-db-w97nj b422fa27d2e9b3a283062590d046b94a31b6b7019bd91bd7a55f8765d1b635a7 network default NAD default], result "{}", err I0329 03:05:22.944070 3827 obj_retry.go:464] Detected object tsf-tpa/create-db-w97nj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:22.944100 3827 pods.go:176] Deleting pod: tsf-tpa/create-db-w97nj I0329 03:05:22.945401 3827 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-w97nj, ips: 10.129.0.39 I0329 03:05:22.945432 3827 obj_retry.go:464] Detected object tsf-tpa/create-db-w97nj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:23.823598 3827 obj_retry.go:458] Detected object tsf-tpa/create-db-w97nj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:23.823619 3827 obj_retry.go:458] Detected object tsf-tpa/create-db-w97nj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:28.510426 3827 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:28.510997 3827 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:29.535245 3827 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:29.535272 3827 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:29.720801 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0329 03:05:30.095602 3827 obj_retry.go:464] Detected object tsf-tpa/migrate-db-c46dv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:30.096117 3827 obj_retry.go:464] Detected object tsf-tpa/migrate-db-c46dv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:30.988485 3827 obj_retry.go:458] Detected object tsf-tpa/migrate-db-c46dv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:30.988507 3827 obj_retry.go:458] Detected object tsf-tpa/migrate-db-c46dv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:34.137561 3827 base_network_controller_pods.go:487] [default/tsf-tpa/server-6666b5554-swncp] creating logical port tsf-tpa_server-6666b5554-swncp for pod on switch ip-10-0-1-37.ec2.internal I0329 03:05:34.137684 3827 kube.go:256] Updating pod tsf-tpa/server-6666b5554-swncp I0329 03:05:34.159245 3827 pod.go:62] [tsf-tpa/server-6666b5554-swncp] pod update took 21.596863ms I0329 03:05:34.159268 3827 base_network_controller_pods.go:937] [default/tsf-tpa/server-6666b5554-swncp] addLogicalPort annotation time took 21.624925ms I0329 03:05:34.160397 3827 pods.go:271] [tsf-tpa/server-6666b5554-swncp] addLogicalPort took 22.861312ms, libovsdb time 771.922µs I0329 03:05:35.563939 3827 obj_retry.go:464] Detected object tsf-tpa/create-importers-v6xfb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:35.564548 3827 obj_retry.go:464] Detected object tsf-tpa/create-importers-v6xfb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:36.024946 3827 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-hxjr7] creating logical port tsf-tpa_importer-74cd48f5c7-hxjr7 for pod on switch ip-10-0-1-37.ec2.internal I0329 03:05:36.025092 3827 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-hxjr7 I0329 03:05:36.044174 3827 pod.go:62] [tsf-tpa/importer-74cd48f5c7-hxjr7] pod update took 19.107117ms I0329 03:05:36.044201 3827 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-hxjr7] addLogicalPort annotation time took 19.143277ms I0329 03:05:36.045452 3827 pods.go:271] [tsf-tpa/importer-74cd48f5c7-hxjr7] addLogicalPort took 20.52357ms, libovsdb time 818.252µs I0329 03:05:36.582198 3827 obj_retry.go:458] Detected object tsf-tpa/create-importers-v6xfb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:36.582229 3827 obj_retry.go:458] Detected object tsf-tpa/create-importers-v6xfb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:37.607762 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=555 I0329 03:05:43.500839 3827 cni.go:403] [tsf-tpa/server-6666b5554-swncp 91775629ae0b05caba07b7fee82cf87ccc9a89d5aed436fdf88680a6caec61d3 network default NAD default] ADD starting CNI request [tsf-tpa/server-6666b5554-swncp 91775629ae0b05caba07b7fee82cf87ccc9a89d5aed436fdf88680a6caec61d3 network default NAD default] I0329 03:05:43.509373 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-6666b5554-swncp, hostIfaceName: 91775629ae0b05c, network: default, NAD default, SandboxID: "91775629ae0b05caba07b7fee82cf87ccc9a89d5aed436fdf88680a6caec61d3", PCI device ID: , UID: "18e225dc-5518-42c2-85ef-d8519c14c87d", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0329 03:05:43.555394 3827 cni.go:424] [tsf-tpa/server-6666b5554-swncp 91775629ae0b05caba07b7fee82cf87ccc9a89d5aed436fdf88680a6caec61d3 network default NAD default] ADD finished CNI request [tsf-tpa/server-6666b5554-swncp 91775629ae0b05caba07b7fee82cf87ccc9a89d5aed436fdf88680a6caec61d3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:93:f0:fc:05:24\",\"name\":\"91775629ae0b05c\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/28ce5752-90b9-435b-befd-e25d29b44981\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0329 03:05:43.586789 3827 cni.go:403] [tsf-tpa/importer-74cd48f5c7-hxjr7 596d75efb742e7d5594a812c3ae147b1f2fbd2cfb2fbc518fb0b94f000a85e0c network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-hxjr7 596d75efb742e7d5594a812c3ae147b1f2fbd2cfb2fbc518fb0b94f000a85e0c network default NAD default] I0329 03:05:43.597098 3827 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-hxjr7, hostIfaceName: 596d75efb742e7d, network: default, NAD default, SandboxID: "596d75efb742e7d5594a812c3ae147b1f2fbd2cfb2fbc518fb0b94f000a85e0c", PCI device ID: , UID: "b1bd4753-8e30-4971-809f-6df26188dfb1", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0329 03:05:43.646204 3827 cni.go:424] [tsf-tpa/importer-74cd48f5c7-hxjr7 596d75efb742e7d5594a812c3ae147b1f2fbd2cfb2fbc518fb0b94f000a85e0c network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-hxjr7 596d75efb742e7d5594a812c3ae147b1f2fbd2cfb2fbc518fb0b94f000a85e0c network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:a5:5f:75:50:67\",\"name\":\"596d75efb742e7d\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67a7b337-ba01-411a-8f02-f6c62730f852\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0329 03:05:43.930752 3827 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0329 03:05:46.479255 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0329 03:05:54.925353 3827 util.go:959] Failed to build node endpoints for node ip-10-0-1-37.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0329 03:06:09.463982 3827 namespace.go:142] [openshift-pipelines] updating namespace I0329 03:06:11.742124 3827 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:11.742714 3827 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:11.784673 3827 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:11.784693 3827 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:11.793196 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0329 03:06:11.793225 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0329 03:06:12.517317 3827 namespace.go:98] [integration-service] adding namespace I0329 03:06:12.518245 3827 namespace.go:102] [integration-service] adding namespace took 909.157µs I0329 03:06:12.521732 3827 namespace.go:98] [image-controller] adding namespace I0329 03:06:12.522317 3827 namespace.go:102] [image-controller] adding namespace took 568.662µs I0329 03:06:12.524945 3827 namespace.go:98] [build-service] adding namespace I0329 03:06:12.525513 3827 namespace.go:102] [build-service] adding namespace took 553.566µs I0329 03:06:12.563516 3827 namespace.go:142] [integration-service] updating namespace I0329 03:06:12.587787 3827 namespace.go:142] [integration-service] updating namespace I0329 03:06:12.593259 3827 namespace.go:142] [image-controller] updating namespace I0329 03:06:12.619656 3827 namespace.go:142] [image-controller] updating namespace I0329 03:06:12.629494 3827 namespace.go:142] [build-service] updating namespace I0329 03:06:12.647693 3827 namespace.go:142] [build-service] updating namespace I0329 03:06:12.716564 3827 namespace.go:142] [build-service] updating namespace I0329 03:06:12.754772 3827 namespace.go:142] [integration-service] updating namespace I0329 03:06:12.801663 3827 namespace.go:98] [release-service] adding namespace I0329 03:06:12.802641 3827 namespace.go:102] [release-service] adding namespace took 959.15µs I0329 03:06:12.863830 3827 namespace.go:142] [release-service] updating namespace I0329 03:06:12.879837 3827 namespace.go:142] [release-service] updating namespace I0329 03:06:12.899107 3827 namespace.go:142] [konflux-ui] updating namespace I0329 03:06:12.986819 3827 namespace.go:98] [konflux-info] adding namespace I0329 03:06:12.987618 3827 namespace.go:102] [konflux-info] adding namespace took 780.436µs I0329 03:06:13.031500 3827 namespace.go:98] [namespace-lister] adding namespace I0329 03:06:13.032236 3827 namespace.go:102] [namespace-lister] adding namespace took 716.401µs I0329 03:06:13.068963 3827 namespace.go:142] [konflux-info] updating namespace I0329 03:06:13.097843 3827 namespace.go:98] [enterprise-contract-service] adding namespace I0329 03:06:13.098617 3827 namespace.go:102] [enterprise-contract-service] adding namespace took 757.904µs I0329 03:06:13.105767 3827 namespace.go:142] [image-controller] updating namespace I0329 03:06:13.152070 3827 namespace.go:142] [konflux-info] updating namespace I0329 03:06:13.218874 3827 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:13.296828 3827 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:13.296852 3827 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:13.374709 3827 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:13.374789 3827 namespace.go:98] [default-tenant] adding namespace I0329 03:06:13.375704 3827 namespace.go:102] [default-tenant] adding namespace took 895.457µs I0329 03:06:13.488950 3827 namespace.go:142] [default-tenant] updating namespace I0329 03:06:13.489148 3827 namespace.go:98] [segment-bridge] adding namespace I0329 03:06:13.489979 3827 namespace.go:102] [segment-bridge] adding namespace took 808.617µs I0329 03:06:13.603721 3827 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:13.603907 3827 namespace.go:142] [default-tenant] updating namespace I0329 03:06:13.749817 3827 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:14.247132 3827 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0329 03:06:14.247778 3827 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0329 03:06:14.247800 3827 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0329 03:06:14.250226 3827 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0329 03:06:14.331855 3827 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0329 03:06:14.331880 3827 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0329 03:06:14.332658 3827 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0329 03:06:14.644875 3827 namespace.go:142] [build-service] updating namespace I0329 03:06:15.139137 3827 namespace.go:142] [image-controller] updating namespace I0329 03:06:15.397983 3827 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:15.398677 3827 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:15.417826 3827 namespace.go:142] [integration-service] updating namespace I0329 03:06:15.625585 3827 namespace.go:142] [konflux-info] updating namespace I0329 03:06:15.826975 3827 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:16.023097 3827 namespace.go:142] [release-service] updating namespace I0329 03:06:16.155899 3827 namespace.go:142] [build-service] updating namespace I0329 03:06:16.171780 3827 namespace.go:142] [image-controller] updating namespace I0329 03:06:16.183750 3827 namespace.go:142] [integration-service] updating namespace I0329 03:06:16.204115 3827 namespace.go:142] [konflux-info] updating namespace I0329 03:06:16.225580 3827 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:16.236594 3827 namespace.go:142] [release-service] updating namespace I0329 03:06:16.283526 3827 namespace.go:142] [build-service] updating namespace I0329 03:06:16.333552 3827 namespace.go:142] [image-controller] updating namespace I0329 03:06:16.396177 3827 namespace.go:142] [integration-service] updating namespace I0329 03:06:16.427482 3827 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:16.427508 3827 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:16.502277 3827 namespace.go:142] [konflux-info] updating namespace I0329 03:06:16.605713 3827 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:16.697382 3827 namespace.go:142] [release-service] updating namespace W0329 03:06:17.288348 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0329 03:06:17.913520 3827 namespace.go:142] [default-tenant] updating namespace I0329 03:06:18.113293 3827 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:18.321532 3827 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:18.439374 3827 namespace.go:142] [default-tenant] updating namespace I0329 03:06:18.458231 3827 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:18.474899 3827 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:18.519453 3827 namespace.go:142] [default-tenant] updating namespace I0329 03:06:18.601432 3827 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:18.691755 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0329 03:06:18.703104 3827 namespace.go:142] [segment-bridge] updating namespace W0329 03:06:20.293877 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0329 03:06:20.315209 3827 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0329 03:06:25.333918 3827 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0329 03:06:29.598812 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=151 W0329 03:06:30.376245 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:30.376269 3827 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:30.409072 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0329 03:06:30.409139 3827 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0329 03:06:30.426270 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0329 03:06:30.426451 3827 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0329 03:06:32.755892 3827 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:32.755912 3827 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:32.779123 3827 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0329 03:06:32.779267 3827 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0329 03:06:44.149841 3827 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:44.150493 3827 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:44.470669 3827 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:44.470695 3827 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:44.480127 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-rbtgb I0329 03:06:44.480151 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-rbtgb I0329 03:07:00.720194 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0329 03:07:07.703166 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0329 03:07:15.595344 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=170 I0329 03:07:29.709212 3827 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0329 03:07:39.365843 3827 obj_retry.go:464] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:39.366442 3827 obj_retry.go:464] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:39.826257 3827 obj_retry.go:458] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:39.826281 3827 obj_retry.go:458] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:39.834380 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5bbc66485b-k45f5 I0329 03:07:39.834409 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5bbc66485b-k45f5 I0329 03:07:54.330610 3827 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:54.331169 3827 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:54.351795 3827 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:54.351819 3827 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:54.357489 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0329 03:07:54.357511 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0329 03:08:19.049734 3827 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default] I0329 03:08:19.137564 3827 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 8639daff6745c83baf1c350467218526fe7a26f18194bf4706d6fffbfbdd187d network default NAD default], result "{}", err I0329 03:08:19.197501 3827 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:19.197543 3827 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 I0329 03:08:19.198986 3827 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6, ips: 10.129.0.23 I0329 03:08:19.199021 3827 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:19.694099 3827 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:19.694126 3827 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:19.702428 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 I0329 03:08:19.702448 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 I0329 03:08:32.410268 3827 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:32.410891 3827 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:33.157000 3827 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:33.157027 3827 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:33.164517 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz I0329 03:08:33.164544 3827 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz I0329 03:09:01.593962 3827 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10