+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-138.ec2.internal ++ [[ -n ip-10-0-1-138.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-138.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0331 03:55:27.398678520 - disable conntrack on geneve port' I0331 03:55:27.398678520 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0331 03:55:27.415606327 - starting ovnkube-node' I0331 03:55:27.415606327 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-138.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-138.ec2.internal --init-node ip-10-0-1-138.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-138.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0331 03:55:27.455414 3792 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0331 03:55:27.455457 3792 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-f465bfb369.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0331 03:55:27.456981 3792 kube.go:419] Waiting for certificate I0331 03:55:27.457009 3792 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0331 03:55:27.457049 3792 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0331 03:55:27.457824 3792 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:55:27.489593 3792 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:55:27.489616 3792 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:55:27.493299 3792 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:55:27.493383 3792 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-6g6sw" I0331 03:55:32.638388 3792 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-6g6sw" I0331 03:55:32.638499 3792 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:55:33.457167 3792 kube.go:426] Certificate found I0331 03:55:33.457706 3792 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:55:33.458564 3792 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0331 03:55:33.525359 3792 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:55:33.541541 3792 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0331 03:55:33.544284 3792 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0331 03:55:33.544359 3792 factory.go:531] Starting watch factory I0331 03:55:33.544451 3792 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544472 3792 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544452 3792 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544502 3792 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544525 3792 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544532 3792 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544531 3792 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544566 3792 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544451 3792 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544894 3792 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.544966 3792 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.545198 3792 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.546028 3792 controller_manager.go:368] Starting the ovnkube controller I0331 03:55:33.546040 3792 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-138.ec2.internal I0331 03:55:33.546125 3792 controller_manager.go:393] NBDB zone sync took: 79.058µs I0331 03:55:33.546137 3792 factory.go:531] Starting watch factory I0331 03:55:33.560029 3792 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.560846 3792 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.561021 3792 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.564623 3792 ovn_northd.go:121] Registering OVN northd metrics I0331 03:55:33.565688 3792 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0331 03:55:33.567446 3792 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.567789 3792 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.569878 3792 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0331 03:55:33.569910 3792 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0331 03:55:33.574963 3792 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:55:33.586805 3792 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0331 03:55:33.645990 3792 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-01 03:50:32 +0000 UTC" deadline="2026-04-01 00:52:33.058392058 +0000 UTC" I0331 03:55:33.646016 3792 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h56m59.412383885s" I0331 03:55:33.650112 3792 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:55:33.650136 3792 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:55:33.650152 3792 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:55:33.650156 3792 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:55:33.650159 3792 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:55:33.650164 3792 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:55:33.650256 3792 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:55:33.650270 3792 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:55:33.650365 3792 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:55:33.650386 3792 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:55:33.652314 3792 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:55:33.652325 3792 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:55:33.652329 3792 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:55:33.652332 3792 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:55:33.652335 3792 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:55:33.652338 3792 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:55:33.653153 3792 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:55:33.653154 3792 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:55:33.655296 3792 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:55:33.655305 3792 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:55:33.655369 3792 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.655384 3792 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.657592 3792 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:55:33.657604 3792 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:55:33.657980 3792 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.661177 3792 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:55:33.661247 3792 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.661277 3792 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.663347 3792 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:55:33.663735 3792 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.666872 3792 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:55:33.666948 3792 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.666965 3792 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.669055 3792 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:55:33.669437 3792 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.672528 3792 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:55:33.672582 3792 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.672597 3792 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.674328 3792 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:55:33.675201 3792 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.678342 3792 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:55:33.678394 3792 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.678405 3792 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.679486 3792 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:55:33.680604 3792 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.683872 3792 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:55:33.684030 3792 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:55:33.684043 3792 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:55:33.685336 3792 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:55:33.686655 3792 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:55:33.689900 3792 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:55:33.689976 3792 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.689993 3792 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.690018 3792 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.690038 3792 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.690540 3792 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:55:33.692506 3792 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.693097 3792 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:55:33.695292 3792 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:55:33.695304 3792 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:55:33.695313 3792 factory.go:636] Watch Factory start up complete, took: 150.956394ms I0331 03:55:33.695406 3792 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0331 03:55:33.695564 3792 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:55:33.695577 3792 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:55:33.695586 3792 factory.go:636] Watch Factory start up complete, took: 149.446928ms I0331 03:55:33.695597 3792 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-138.ec2.internal" zone I0331 03:55:33.730469 3792 default_node_network_controller.go:709] Initializing the default node network controller I0331 03:55:33.782951 3792 udn_isolation.go:101] Starting UDN host isolation manager I0331 03:55:33.793532 3792 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0331 03:55:33.838960 3792 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0331 03:55:33.839037 3792 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0331 03:55:33.839052 3792 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0331 03:55:33.872968 3792 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0331 03:55:33.873040 3792 default_node_network_controller.go:808] Node ip-10-0-1-138.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0331 03:55:33.878960 3792 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.138"] k8s.ovn.org/zone-name:ip-10-0-1-138.ec2.internal] on node ip-10-0-1-138.ec2.internal I0331 03:55:33.898868 3792 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0331 03:55:33.904507 3792 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0331 03:55:33.904726 3792 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0331 03:55:33.904889 3792 gateway_init.go:263] Preparing Gateway I0331 03:55:33.904904 3792 gateway_shared_intf.go:1658] Creating new gateway I0331 03:55:33.946311 3792 controller_manager.go:427] Waiting for node in zone sync took: 250.698836ms I0331 03:55:33.951087 3792 controller_manager.go:608] Switching to transit router for layer2 networks I0331 03:55:33.951102 3792 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-138.ec2.internal I0331 03:55:33.970073 3792 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:55:33.975796 3792 controller_manager.go:288] SCTP support detected in OVN I0331 03:55:33.976454 3792 services_controller.go:69] Creating services controller for network=default I0331 03:55:33.976544 3792 udn_enabled_svc.go:80] Waiting for service informer to sync I0331 03:55:33.976605 3792 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0331 03:55:33.976634 3792 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0331 03:55:33.976650 3792 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0331 03:55:33.976662 3792 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:55:33.976808 3792 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0331 03:55:33.976848 3792 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0331 03:55:33.976886 3792 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 27.473µs I0331 03:55:33.976890 3792 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 59.999µs I0331 03:55:33.976896 3792 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 48.175µs I0331 03:55:33.976903 3792 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0331 03:55:33.976912 3792 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0331 03:55:33.976925 3792 default_network_controller.go:321] Starting the default network controller I0331 03:55:33.977023 3792 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 26.359µs I0331 03:55:33.977109 3792 udn_enabled_svc.go:95] Performing full resync I0331 03:55:33.977439 3792 udn_enabled_svc.go:99] Waiting for handler to sync I0331 03:55:33.977455 3792 udn_enabled_svc.go:104] Starting worker I0331 03:55:33.979547 3792 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:55:33.985968 3792 iptables.go:108] Creating table: filter chain: OUTPUT I0331 03:55:33.986064 3792 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0331 03:55:33.986132 3792 repair.go:33] Syncing exgw routes took 54.617µs I0331 03:55:33.986168 3792 default_network_controller.go:407] Starting all the Watchers... I0331 03:55:33.986337 3792 namespace.go:98] [openshift-network-console] adding namespace I0331 03:55:33.986349 3792 namespace.go:98] [openshift-apiserver] adding namespace I0331 03:55:33.986350 3792 namespace.go:98] [openshift-logging] adding namespace I0331 03:55:33.986360 3792 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0331 03:55:33.986365 3792 namespace.go:98] [openshift-apiserver-operator] adding namespace I0331 03:55:33.986376 3792 namespace.go:98] [openshift-config] adding namespace I0331 03:55:33.986379 3792 namespace.go:98] [openshift-service-ca-operator] adding namespace I0331 03:55:33.986385 3792 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0331 03:55:33.986394 3792 namespace.go:98] [openshift-operators-redhat] adding namespace I0331 03:55:33.986400 3792 namespace.go:98] [openshift-image-registry] adding namespace I0331 03:55:33.986403 3792 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0331 03:55:33.986407 3792 namespace.go:98] [openshift-backplane-srep] adding namespace I0331 03:55:33.986410 3792 namespace.go:98] [openshift-backplane-tam] adding namespace I0331 03:55:33.986414 3792 namespace.go:98] [openshift-service-ca] adding namespace I0331 03:55:33.986419 3792 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0331 03:55:33.986908 3792 namespace.go:102] [openshift-network-console] adding namespace took 561.118µs I0331 03:55:33.986922 3792 namespace.go:98] [openshift-monitoring] adding namespace I0331 03:55:33.987476 3792 namespace.go:102] [openshift-apiserver] adding namespace took 1.117217ms I0331 03:55:33.987499 3792 namespace.go:98] [default] adding namespace I0331 03:55:33.988106 3792 namespace.go:102] [default] adding namespace took 598.626µs I0331 03:55:33.988124 3792 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0331 03:55:33.988753 3792 namespace.go:102] [openshift-logging] adding namespace took 2.390157ms I0331 03:55:33.988771 3792 namespace.go:98] [openshift-kube-scheduler] adding namespace I0331 03:55:33.989471 3792 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 3.101345ms I0331 03:55:33.989488 3792 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0331 03:55:33.990094 3792 namespace.go:102] [openshift-apiserver-operator] adding namespace took 3.716263ms I0331 03:55:33.990111 3792 namespace.go:98] [openshift-host-network] adding namespace I0331 03:55:33.990826 3792 namespace.go:102] [openshift-config] adding namespace took 4.43701ms I0331 03:55:33.990847 3792 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0331 03:55:33.991593 3792 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.202429ms I0331 03:55:33.991608 3792 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0331 03:55:33.992176 3792 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.778204ms I0331 03:55:33.992201 3792 namespace.go:98] [openshift-route-controller-manager] adding namespace I0331 03:55:33.992500 3792 gateway_shared_intf.go:1787] Gateway Creation Complete I0331 03:55:33.992519 3792 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-138.ec2.internal","mac-address":"0a:ff:c1:99:bb:a3","ip-addresses":["10.0.1.138/24"],"ip-address":"10.0.1.138/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f1ef321b-f04f-449f-833e-8a3095de6d59 k8s.ovn.org/node-encap-ips:["10.0.1.138"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.138/24"} k8s.ovn.org/zone-name:ip-10-0-1-138.ec2.internal] on node ip-10-0-1-138.ec2.internal I0331 03:55:33.992684 3792 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.282346ms I0331 03:55:33.992703 3792 namespace.go:98] [openshift-console-user-settings] adding namespace I0331 03:55:33.993282 3792 namespace.go:102] [openshift-image-registry] adding namespace took 6.876207ms I0331 03:55:33.993299 3792 namespace.go:98] [openshift-console-operator] adding namespace I0331 03:55:33.993867 3792 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.459167ms I0331 03:55:33.993879 3792 namespace.go:98] [openshift-kube-apiserver] adding namespace I0331 03:55:33.994410 3792 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.996964ms I0331 03:55:33.994426 3792 namespace.go:98] [openshift-node] adding namespace I0331 03:55:33.994921 3792 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.505398ms I0331 03:55:33.994937 3792 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0331 03:55:33.995545 3792 namespace.go:102] [openshift-service-ca] adding namespace took 9.124865ms I0331 03:55:33.995564 3792 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0331 03:55:33.996048 3792 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 9.624471ms I0331 03:55:33.996062 3792 namespace.go:98] [openshift-config-managed] adding namespace I0331 03:55:33.996625 3792 namespace.go:102] [openshift-monitoring] adding namespace took 9.697513ms I0331 03:55:33.996640 3792 namespace.go:98] [openshift-authentication] adding namespace I0331 03:55:33.997073 3792 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.944145ms I0331 03:55:33.997085 3792 namespace.go:98] [openshift-marketplace] adding namespace I0331 03:55:33.997700 3792 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.921418ms I0331 03:55:33.997716 3792 namespace.go:98] [openshift-package-operator] adding namespace I0331 03:55:33.998171 3792 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.677358ms I0331 03:55:33.998184 3792 namespace.go:98] [openshift-network-operator] adding namespace I0331 03:55:33.998698 3792 namespace.go:102] [openshift-host-network] adding namespace took 8.579083ms I0331 03:55:33.998715 3792 namespace.go:98] [openshift-ingress-canary] adding namespace I0331 03:55:33.999283 3792 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 8.426989ms I0331 03:55:33.999298 3792 namespace.go:98] [openshift] adding namespace I0331 03:55:33.999748 3792 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 8.133018ms I0331 03:55:33.999762 3792 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0331 03:55:34.000187 3792 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.976605ms I0331 03:55:34.000207 3792 namespace.go:98] [openshift-backplane-cee] adding namespace I0331 03:55:34.000759 3792 namespace.go:102] [openshift-console-user-settings] adding namespace took 8.033642ms I0331 03:55:34.000777 3792 namespace.go:98] [openshift-ingress] adding namespace I0331 03:55:34.001295 3792 namespace.go:102] [openshift-console-operator] adding namespace took 7.988568ms I0331 03:55:34.001312 3792 namespace.go:98] [openshift-backplane] adding namespace I0331 03:55:34.001748 3792 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.864083ms I0331 03:55:34.001762 3792 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0331 03:55:34.002295 3792 namespace.go:102] [openshift-node] adding namespace took 7.863543ms I0331 03:55:34.002313 3792 namespace.go:98] [openshift-ingress-operator] adding namespace I0331 03:55:34.002911 3792 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.966928ms I0331 03:55:34.002929 3792 namespace.go:98] [openshift-network-node-identity] adding namespace I0331 03:55:34.003456 3792 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.88622ms I0331 03:55:34.003472 3792 namespace.go:98] [openshift-must-gather-operator] adding namespace I0331 03:55:34.003924 3792 namespace.go:102] [openshift-config-managed] adding namespace took 7.85468ms I0331 03:55:34.003941 3792 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0331 03:55:34.004446 3792 namespace.go:102] [openshift-authentication] adding namespace took 7.798044ms I0331 03:55:34.004461 3792 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0331 03:55:34.004912 3792 namespace.go:102] [openshift-marketplace] adding namespace took 7.819808ms I0331 03:55:34.004928 3792 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0331 03:55:34.005527 3792 namespace.go:102] [openshift-package-operator] adding namespace took 7.803568ms I0331 03:55:34.005545 3792 namespace.go:98] [openshift-console] adding namespace I0331 03:55:34.006044 3792 namespace.go:102] [openshift-network-operator] adding namespace took 7.853543ms I0331 03:55:34.006064 3792 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0331 03:55:34.006631 3792 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.910334ms I0331 03:55:34.006646 3792 namespace.go:98] [openshift-operators] adding namespace I0331 03:55:34.007159 3792 namespace.go:102] [openshift] adding namespace took 7.854996ms I0331 03:55:34.007178 3792 namespace.go:98] [openshift-infra] adding namespace I0331 03:55:34.007612 3792 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.842511ms I0331 03:55:34.007631 3792 namespace.go:98] [openshift-etcd] adding namespace I0331 03:55:34.008802 3792 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.587891ms I0331 03:55:34.008819 3792 namespace.go:98] [openshift-machine-config-operator] adding namespace I0331 03:55:34.009416 3792 namespace.go:102] [openshift-ingress] adding namespace took 8.633564ms I0331 03:55:34.009429 3792 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0331 03:55:34.009886 3792 namespace.go:102] [openshift-backplane] adding namespace took 8.566129ms I0331 03:55:34.009907 3792 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0331 03:55:34.010615 3792 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 8.845912ms I0331 03:55:34.010635 3792 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0331 03:55:34.011168 3792 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.848595ms I0331 03:55:34.011183 3792 namespace.go:98] [kube-node-lease] adding namespace I0331 03:55:34.011742 3792 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.806914ms I0331 03:55:34.011760 3792 namespace.go:98] [openshift-authentication-operator] adding namespace I0331 03:55:34.011836 3792 default_node_network_controller.go:518] Node connection status = connected I0331 03:55:34.012310 3792 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.829202ms I0331 03:55:34.012327 3792 namespace.go:98] [openshift-controller-manager] adding namespace I0331 03:55:34.012870 3792 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.921961ms I0331 03:55:34.012887 3792 namespace.go:98] [openshift-backplane-cse] adding namespace I0331 03:55:34.013516 3792 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 9.048399ms I0331 03:55:34.013530 3792 namespace.go:98] [openshift-customer-monitoring] adding namespace I0331 03:55:34.014058 3792 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 9.123456ms I0331 03:55:34.014711 3792 namespace.go:102] [openshift-console] adding namespace took 9.158276ms I0331 03:55:34.014727 3792 namespace.go:98] [openshift-cluster-version] adding namespace I0331 03:55:34.015274 3792 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 9.200819ms I0331 03:55:34.015295 3792 namespace.go:98] [kube-system] adding namespace I0331 03:55:34.015844 3792 namespace.go:102] [openshift-operators] adding namespace took 9.18912ms I0331 03:55:34.015864 3792 namespace.go:98] [openshift-backplane-mobb] adding namespace I0331 03:55:34.016410 3792 namespace.go:102] [openshift-infra] adding namespace took 9.225458ms I0331 03:55:34.016427 3792 namespace.go:98] [openshift-network-diagnostics] adding namespace I0331 03:55:34.016954 3792 namespace.go:102] [openshift-etcd] adding namespace took 9.316197ms I0331 03:55:34.016971 3792 namespace.go:98] [kube-public] adding namespace I0331 03:55:34.017596 3792 namespace.go:102] [openshift-machine-config-operator] adding namespace took 8.769971ms I0331 03:55:34.017614 3792 namespace.go:98] [openshift-dns] adding namespace I0331 03:55:34.018313 3792 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 8.877223ms I0331 03:55:34.018329 3792 namespace.go:98] [openshift-machine-api] adding namespace I0331 03:55:34.018933 3792 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 9.018356ms I0331 03:55:34.018952 3792 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0331 03:55:34.019642 3792 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 8.99845ms I0331 03:55:34.019657 3792 namespace.go:98] [openshift-insights] adding namespace I0331 03:55:34.020312 3792 namespace.go:102] [kube-node-lease] adding namespace took 9.122965ms I0331 03:55:34.020326 3792 namespace.go:98] [openshift-backplane-csm] adding namespace I0331 03:55:34.020981 3792 namespace.go:102] [openshift-authentication-operator] adding namespace took 9.211859ms I0331 03:55:34.021001 3792 namespace.go:98] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] adding namespace I0331 03:55:34.021570 3792 namespace.go:102] [openshift-controller-manager] adding namespace took 9.237183ms I0331 03:55:34.021583 3792 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0331 03:55:34.022102 3792 namespace.go:102] [openshift-backplane-cse] adding namespace took 9.209285ms I0331 03:55:34.022114 3792 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0331 03:55:34.022569 3792 namespace.go:102] [openshift-customer-monitoring] adding namespace took 9.032347ms I0331 03:55:34.022586 3792 namespace.go:98] [dedicated-admin] adding namespace I0331 03:55:34.023109 3792 namespace.go:102] [openshift-cluster-version] adding namespace took 8.375884ms I0331 03:55:34.023136 3792 namespace.go:98] [openshift-dns-operator] adding namespace W0331 03:55:34.023102 3792 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-9ktxq: pod kube-system/global-pull-secret-syncer-9ktxq: no pod IPs found I0331 03:55:34.023717 3792 namespace.go:102] [kube-system] adding namespace took 8.41526ms I0331 03:55:34.023731 3792 namespace.go:98] [openshift-controller-manager-operator] adding namespace W0331 03:55:34.024123 3792 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-st94r: pod openshift-network-diagnostics/network-check-target-st94r: no pod IPs found W0331 03:55:34.024138 3792 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-t4kxf: pod openshift-network-diagnostics/network-check-target-t4kxf: no pod IPs found I0331 03:55:34.024173 3792 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.301092ms I0331 03:55:34.024681 3792 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.246732ms I0331 03:55:34.025119 3792 namespace.go:102] [kube-public] adding namespace took 8.140308ms I0331 03:55:34.025558 3792 namespace.go:102] [openshift-dns] adding namespace took 7.939421ms I0331 03:55:34.025572 3792 namespace.go:98] [openshift-config-operator] adding namespace I0331 03:55:34.026061 3792 namespace.go:102] [openshift-machine-api] adding namespace took 7.726702ms I0331 03:55:34.026619 3792 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.659108ms I0331 03:55:34.027184 3792 namespace.go:102] [openshift-insights] adding namespace took 7.520272ms I0331 03:55:34.027204 3792 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0331 03:55:34.027705 3792 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.370028ms I0331 03:55:34.028224 3792 namespace.go:102] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] adding namespace took 7.216762ms I0331 03:55:34.028240 3792 namespace.go:98] [openshift-multus] adding namespace I0331 03:55:34.028734 3792 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.145476ms I0331 03:55:34.029171 3792 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.0511ms I0331 03:55:34.029759 3792 namespace.go:102] [dedicated-admin] adding namespace took 7.163682ms I0331 03:55:34.030264 3792 namespace.go:102] [openshift-dns-operator] adding namespace took 7.12044ms I0331 03:55:34.030805 3792 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.068722ms I0331 03:55:34.031430 3792 namespace.go:102] [openshift-config-operator] adding namespace took 5.85174ms W0331 03:55:34.031855 3792 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-64dxv: pod openshift-multus/network-metrics-daemon-64dxv: no pod IPs found W0331 03:55:34.031868 3792 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-nwlvp: pod openshift-multus/network-metrics-daemon-nwlvp: no pod IPs found I0331 03:55:34.031906 3792 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 4.695395ms I0331 03:55:34.032426 3792 namespace.go:102] [openshift-multus] adding namespace took 4.180528ms I0331 03:55:34.032825 3792 zone_ic_handler.go:186] Time taken to create transit switch: 292.717µs I0331 03:55:34.032911 3792 master.go:532] Adding or Updating local node "ip-10-0-1-138.ec2.internal" for network "default" I0331 03:55:34.032959 3792 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-60.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-60.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-60.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-60.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-60.ec2.internal I0331 03:55:34.033602 3792 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-188.ec2.internal for the network default I0331 03:55:34.034613 3792 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-188.ec2.internal" on network "default" took: 1.024932ms W0331 03:55:34.042616 3792 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-st94r: pod openshift-network-diagnostics/network-check-target-st94r: no pod IPs found W0331 03:55:34.042635 3792 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-nwlvp: pod openshift-multus/network-metrics-daemon-nwlvp: no pod IPs found I0331 03:55:34.047606 3792 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-138.ec2.internal for the network default I0331 03:55:34.048808 3792 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0331 03:55:34.048903 3792 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0331 03:55:34.048923 3792 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0331 03:55:34.048930 3792 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:55:34.048972 3792 node_tracker.go:235] Node ip-10-0-0-60.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-60.ec2.internal" I0331 03:55:34.048994 3792 node_tracker.go:169] Node ip-10-0-0-60.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.049004 3792 services_controller.go:574] Full service sync requested for network=default I0331 03:55:34.049012 3792 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-138.ec2.internal in network "default" I0331 03:55:34.049055 3792 node_tracker.go:169] Node ip-10-0-1-138.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.049100 3792 services_controller.go:574] Full service sync requested for network=default I0331 03:55:34.049117 3792 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-188.ec2.internal in network "default" I0331 03:55:34.049188 3792 node_tracker.go:169] Node ip-10-0-2-188.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.049199 3792 services_controller.go:574] Full service sync requested for network=default I0331 03:55:34.104400 3792 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:55:34.104429 3792 default_network_controller.go:1008] Node "ip-10-0-0-60.ec2.internal" in remote zone "ip-10-0-0-60.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:55:34.104459 3792 node_tracker.go:169] Node ip-10-0-0-60.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.104473 3792 services_controller.go:574] Full service sync requested for network=default I0331 03:55:34.105130 3792 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-60.ec2.internal for the network default I0331 03:55:34.106452 3792 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-60.ec2.internal" on network "default" took: 1.341267ms I0331 03:55:34.149990 3792 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0331 03:55:34.150015 3792 services_controller.go:208] Setting up event handlers for services for network=default I0331 03:55:34.150085 3792 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0331 03:55:34.150134 3792 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0331 03:55:34.150170 3792 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0331 03:55:34.250566 3792 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0331 03:55:34.250586 3792 repair.go:60] Starting repairing loop for services I0331 03:55:34.250703 3792 repair.go:131] Deleted 0 stale service LBs I0331 03:55:34.250723 3792 repair.go:137] Deleted 0 stale Chassis Template Vars I0331 03:55:34.250738 3792 repair.go:62] Finished repairing loop for services: 153.594µs I0331 03:55:34.250761 3792 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0331 03:55:34.250771 3792 services_controller.go:254] Starting workers for network=default W0331 03:55:34.250916 3792 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:34.251363 3792 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-trkrq of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0331 03:55:34.251427 3792 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-9ktxq, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-9ktxq: suppressed error logged: pod kube-system/global-pull-secret-syncer-9ktxq: no pod IPs found I0331 03:55:34.251454 3792 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-64dxv, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-64dxv: suppressed error logged: pod openshift-multus/network-metrics-daemon-64dxv: no pod IPs found I0331 03:55:34.251582 3792 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-nwlvp] creating logical port openshift-multus_network-metrics-daemon-nwlvp for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:34.251593 3792 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-t4kxf, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-t4kxf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-t4kxf: no pod IPs found I0331 03:55:34.251771 3792 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-st94r] creating logical port openshift-network-diagnostics_network-check-target-st94r for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:34.251790 3792 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-nwlvp I0331 03:55:34.251942 3792 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-st94r W0331 03:55:34.254807 3792 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.254830 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.254836 3792 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0331 03:55:34.268103 3792 pod.go:62] [openshift-multus/network-metrics-daemon-nwlvp] pod update took 16.460512ms I0331 03:55:34.268132 3792 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-nwlvp] addLogicalPort annotation time took 16.492247ms I0331 03:55:34.268354 3792 pod.go:62] [openshift-network-diagnostics/network-check-target-st94r] pod update took 16.465602ms I0331 03:55:34.268376 3792 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-st94r] addLogicalPort annotation time took 16.487303ms I0331 03:55:34.269365 3792 pods.go:271] [openshift-multus/network-metrics-daemon-nwlvp] addLogicalPort took 17.785757ms, libovsdb time 936.486µs I0331 03:55:34.269552 3792 pods.go:271] [openshift-network-diagnostics/network-check-target-st94r] addLogicalPort took 17.790631ms, libovsdb time 966.921µs I0331 03:55:34.270433 3792 admin_network_policy_controller.go:227] Starting controller default-network-controller I0331 03:55:34.270462 3792 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0331 03:55:34.270474 3792 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0331 03:55:34.270479 3792 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0331 03:55:34.270672 3792 repair.go:29] Repairing admin network policies took 167.191µs I0331 03:55:34.270808 3792 repair.go:92] Repairing baseline admin network policies took 123.945µs I0331 03:55:34.271481 3792 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0331 03:55:34.271511 3792 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0331 03:55:34.274298 3792 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0331 03:55:34.274319 3792 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0331 03:55:34.274334 3792 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0331 03:55:34.276408 3792 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0331 03:55:34.276435 3792 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0331 03:55:34.276450 3792 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0331 03:55:34.277958 3792 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0331 03:55:34.277986 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0331 03:55:34.278866 3792 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0331 03:55:34.278881 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:55:34.281264 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0331 03:55:34.281286 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0331 03:55:34.281294 3792 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0331 03:55:34.283017 3792 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0331 03:55:34.283044 3792 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0331 03:55:34.283054 3792 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0331 03:55:34.285323 3792 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0331 03:55:34.285348 3792 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0331 03:55:34.285357 3792 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0331 03:55:34.287607 3792 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0331 03:55:34.287627 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0331 03:55:34.287634 3792 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0331 03:55:34.287974 3792 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0331 03:55:34.287994 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0331 03:55:34.288025 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:55:34.290320 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0331 03:55:34.290343 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0331 03:55:34.290350 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0331 03:55:34.290696 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0331 03:55:34.290711 3792 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0331 03:55:34.290716 3792 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0331 03:55:34.291177 3792 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0331 03:55:34.291209 3792 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0331 03:55:34.291219 3792 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0331 03:55:34.292374 3792 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0331 03:55:34.292394 3792 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0331 03:55:34.292401 3792 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0331 03:55:34.292735 3792 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0331 03:55:34.292759 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0331 03:55:34.292768 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0331 03:55:34.293402 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0331 03:55:34.293426 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0331 03:55:34.293438 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0331 03:55:34.294167 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:55:34.294190 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0331 03:55:34.294199 3792 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0331 03:55:34.295068 3792 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0331 03:55:34.295092 3792 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0331 03:55:34.295105 3792 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0331 03:55:34.296338 3792 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0331 03:55:34.296360 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0331 03:55:34.296367 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0331 03:55:34.296887 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0331 03:55:34.296902 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0331 03:55:34.296910 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0331 03:55:34.297553 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:55:34.297576 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0331 03:55:34.297587 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0331 03:55:34.299533 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0331 03:55:34.299555 3792 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0331 03:55:34.299567 3792 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0331 03:55:34.300135 3792 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0331 03:55:34.300171 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0331 03:55:34.300179 3792 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0331 03:55:34.300798 3792 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0331 03:55:34.300818 3792 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0331 03:55:34.300826 3792 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0331 03:55:34.301378 3792 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0331 03:55:34.301400 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0331 03:55:34.301409 3792 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0331 03:55:34.302022 3792 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0331 03:55:34.302055 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0331 03:55:34.302070 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0331 03:55:34.302665 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0331 03:55:34.302690 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0331 03:55:34.302702 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0331 03:55:34.303266 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0331 03:55:34.303288 3792 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0331 03:55:34.303298 3792 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0331 03:55:34.303946 3792 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0331 03:55:34.303963 3792 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0331 03:55:34.303972 3792 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0331 03:55:34.305233 3792 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0331 03:55:34.305255 3792 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0331 03:55:34.305274 3792 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0331 03:55:34.306540 3792 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0331 03:55:34.306748 3792 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-trkrq of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0331 03:55:34.312678 3792 egressfirewall.go:319] Starting EgressFirewall controller I0331 03:55:34.312708 3792 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0331 03:55:34.312737 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0331 03:55:34.312744 3792 controller.go:133] Adding controller egress-firewall-controller event handlers I0331 03:55:34.312750 3792 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0331 03:55:34.312796 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0331 03:55:34.312808 3792 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0331 03:55:34.313108 3792 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0331 03:55:34.313123 3792 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0331 03:55:34.313133 3792 egressqos.go:193] Setting up event handlers for EgressQoS I0331 03:55:34.313168 3792 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall I0331 03:55:34.313183 3792 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 03:55:34.313187 3792 egressfirewall.go:628] Syncing node "ip-10-0-2-188.ec2.internal" for egress firewall I0331 03:55:34.313263 3792 egressqos.go:245] Starting EgressQoS Controller I0331 03:55:34.313278 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0331 03:55:34.313288 3792 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0331 03:55:34.313296 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0331 03:55:34.313303 3792 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0331 03:55:34.313308 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0331 03:55:34.313312 3792 shared_informer.go:356] "Caches are synced" controller="egressqos" I0331 03:55:34.313316 3792 egressqos.go:259] Repairing EgressQoSes I0331 03:55:34.313320 3792 egressqos.go:399] Starting repairing loop for egressqos I0331 03:55:34.313396 3792 egressqos.go:401] Finished repairing loop for egressqos: 75.842µs I0331 03:55:34.313411 3792 egressservice_zone.go:132] Setting up event handlers for Egress Services I0331 03:55:34.313447 3792 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-138.ec2.internal I0331 03:55:34.313494 3792 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-138.ec2.internal : 49.013µs I0331 03:55:34.313637 3792 egressservice_zone.go:210] Starting Egress Services Controller I0331 03:55:34.313657 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:55:34.313671 3792 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:55:34.313679 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:55:34.313687 3792 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:55:34.313696 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:55:34.313703 3792 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:55:34.313712 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0331 03:55:34.313719 3792 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0331 03:55:34.313726 3792 egressservice_zone.go:228] Repairing Egress Services I0331 03:55:34.314074 3792 master_controller.go:88] Starting Admin Policy Based Route Controller I0331 03:55:34.314083 3792 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:55:34.314098 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:34.314112 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 16.213µs I0331 03:55:34.314119 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:34.314123 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 4.488µs I0331 03:55:34.314128 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-188.ec2.internal I0331 03:55:34.314137 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-188.ec2.internal: 9.121µs I0331 03:55:34.314270 3792 default_network_controller.go:568] Completing all the Watchers took 328.090479ms I0331 03:55:34.314282 3792 default_network_controller.go:572] Starting unidling controllers I0331 03:55:34.314294 3792 unidle.go:46] Registering OVN SB ControllerEvent handler I0331 03:55:34.314299 3792 unidle.go:63] Populating Initial ContollerEvent events I0331 03:55:34.314311 3792 unidle.go:79] Setting up event handlers for services I0331 03:55:34.314370 3792 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:55:34.319733 3792 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:55:34.512635 3792 default_node_network_controller.go:518] Node connection status = connected I0331 03:55:34.529589 3792 bridgeconfig.go:537] Gateway is ready I0331 03:55:34.529609 3792 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0331 03:55:34.540325 3792 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.138. Updating OVN encap IP. I0331 03:55:34.545567 3792 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.138 - it is already configured I0331 03:55:34.545585 3792 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.138/24:{} 172.20.0.1/32:{}]. Updating annotations. I0331 03:55:34.545830 3792 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.138/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-138.ec2.internal","mac-address":"0a:ff:c1:99:bb:a3","ip-addresses":["10.0.1.138/24"],"ip-address":"10.0.1.138/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f1ef321b-f04f-449f-833e-8a3095de6d59 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.138/24"}] on node ip-10-0-1-138.ec2.internal I0331 03:55:34.560938 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:34.560957 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 22.495µs I0331 03:55:34.560944 3792 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-138.ec2.internal in network "default" I0331 03:55:34.560972 3792 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 03:55:34.561056 3792 master.go:532] Adding or Updating local node "ip-10-0-1-138.ec2.internal" for network "default" I0331 03:55:34.561062 3792 node_tracker.go:169] Node ip-10-0-1-138.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.561179 3792 services_controller.go:574] Full service sync requested for network=default I0331 03:55:34.561337 3792 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.138 I0331 03:55:34.561459 3792 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-138.ec2.internal W0331 03:55:34.561610 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.561626 3792 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.561632 3792 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.561685 3792 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:34.566189 3792 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-138.ec2.internal. Updating no re-route policies I0331 03:55:34.566280 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nwlvp I0331 03:55:34.566293 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-tgmkq I0331 03:55:34.566301 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nwlvp I0331 03:55:34.566310 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-tgmkq I0331 03:55:34.566328 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-pmpq6 I0331 03:55:34.566346 3792 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-nwlvp] creating logical port openshift-multus_network-metrics-daemon-nwlvp for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:34.566350 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qgmk2 I0331 03:55:34.566353 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-pmpq6 I0331 03:55:34.566350 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-7wzv6 I0331 03:55:34.566361 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qgmk2 I0331 03:55:34.566359 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-st94r I0331 03:55:34.566368 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-7wzv6 I0331 03:55:34.566373 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ffmzc I0331 03:55:34.566378 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-pmpq6 after 0 failed attempt(s) I0331 03:55:34.566382 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qgmk2 after 0 failed attempt(s) I0331 03:55:34.566388 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ffmzc I0331 03:55:34.566392 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-7wzv6 after 0 failed attempt(s) I0331 03:55:34.566399 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-7v9vg I0331 03:55:34.566398 3792 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-138.ec2.internal I0331 03:55:34.566402 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-9s652 I0331 03:55:34.566406 3792 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-dkwtx I0331 03:55:34.566410 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-5q6bd I0331 03:55:34.566414 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ffmzc after 0 failed attempt(s) I0331 03:55:34.566418 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-9s652 I0331 03:55:34.566419 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-5q6bd I0331 03:55:34.566439 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-9s652 after 0 failed attempt(s) I0331 03:55:34.566418 3792 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-dkwtx I0331 03:55:34.566439 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-5q6bd after 0 failed attempt(s) I0331 03:55:34.566337 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-tgmkq after 0 failed attempt(s) I0331 03:55:34.566405 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-7v9vg I0331 03:55:34.566465 3792 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-dkwtx after 0 failed attempt(s) I0331 03:55:34.566395 3792 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-138.ec2.internal I0331 03:55:34.566471 3792 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-138.ec2.internal I0331 03:55:34.566480 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-138.ec2.internal I0331 03:55:34.566485 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-7v9vg after 0 failed attempt(s) I0331 03:55:34.566498 3792 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-138.ec2.internal after 0 failed attempt(s) I0331 03:55:34.566376 3792 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-st94r I0331 03:55:34.566498 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-138.ec2.internal after 0 failed attempt(s) I0331 03:55:34.566530 3792 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-st94r] creating logical port openshift-network-diagnostics_network-check-target-st94r for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:34.567648 3792 pods.go:271] [openshift-multus/network-metrics-daemon-nwlvp] addLogicalPort took 1.313139ms, libovsdb time 331.717µs I0331 03:55:34.567661 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-nwlvp after 0 failed attempt(s) I0331 03:55:34.567752 3792 pods.go:271] [openshift-network-diagnostics/network-check-target-st94r] addLogicalPort took 1.232273ms, libovsdb time 396.425µs I0331 03:55:34.567766 3792 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-st94r after 0 failed attempt(s) I0331 03:55:34.575853 3792 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0331 03:55:34.587195 3792 iptables.go:108] Creating table: mangle chain: OUTPUT I0331 03:55:34.590692 3792 iptables.go:108] Creating table: nat chain: OUTPUT I0331 03:55:34.618174 3792 iptables.go:108] Creating table: nat chain: PREROUTING I0331 03:55:34.654499 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:34.654506 3792 egressfirewall.go:628] Syncing node "ip-10-0-0-60.ec2.internal" for egress firewall I0331 03:55:34.654519 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 25.249µs I0331 03:55:34.654524 3792 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-60.ec2.internal in network "default" I0331 03:55:34.654539 3792 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-60.ec2.internal. Updating no re-route policies I0331 03:55:34.654573 3792 node_tracker.go:169] Node ip-10-0-0-60.ec2.internal switch + router changed, syncing services in network "default" I0331 03:55:34.654588 3792 services_controller.go:574] Full service sync requested for network=default W0331 03:55:34.654977 3792 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.654991 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.655002 3792 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:34.655093 3792 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:55:34.707616 3792 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:55:34.724639 3792 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0331 03:55:34.724848 3792 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0331 03:55:34.724922 3792 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0331 03:55:34.725009 3792 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0331 03:55:34.725084 3792 gateway_init.go:325] Gateway and management port readiness took 716.643045ms I0331 03:55:34.725115 3792 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0331 03:55:34.725198 3792 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0331 03:55:34.725220 3792 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0331 03:55:34.725237 3792 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:55:34.725315 3792 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0331 03:55:34.725395 3792 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 51.021µs I0331 03:55:34.725729 3792 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0331 03:55:34.725761 3792 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 20.082µs I0331 03:55:34.725772 3792 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 42.342µs I0331 03:55:34.725781 3792 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0331 03:55:34.725795 3792 nad_controller.go:173] [node-nad-controller NAD controller]: started I0331 03:55:34.725802 3792 default_node_network_controller.go:898] Starting the default node network controller I0331 03:55:34.725809 3792 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0331 03:55:34.725879 3792 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 16.49µs I0331 03:55:34.726011 3792 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0331 03:55:34.726023 3792 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0331 03:55:34.726052 3792 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c84070 0xc0002ec4d0 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0002ec690 0xc0002ec700 0xc000baee10 0xc0002ec770 0xc0002ec850 0xc0008a9590 0xc00653efc0 0xc0002ec8c0 map[0x2f67040:0xc002a07020 0x2f67580:0xc002a07260 0x2f68000:0xc002a06f60 0x2f682a0:0xc002a06fc0 0x2f687e0:0xc002a07080 0x2f68a80:0xc002a070e0 0x2f68d20:0xc002a07140 0x2f69500:0xc002a072c0 0x2f697a0:0xc002a07320 0x2f93d40:0xc0009f9bc0 0x2f940e0:0xc0009f9c20 0x2f94bc0:0xc000c86120 0x2f96520:0xc001bc4000 0x2f97000:0xc002a06f00 0x2f97e80:0xc0009f9b60] 0xc000168690 0} I0331 03:55:34.726119 3792 gateway.go:151] Starting gateway service sync I0331 03:55:34.726702 3792 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0331 03:55:34.728847 3792 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0331 03:55:34.759931 3792 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0331 03:55:34.761137 3792 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0331 03:55:34.762359 3792 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0331 03:55:34.797428 3792 gateway.go:168] Gateway service sync done. Time taken: 71.297041ms I0331 03:55:34.798117 3792 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32442 I0331 03:55:34.798250 3792 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c84070 0xc0002ec4d0 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0002ec690 0xc0002ec700 0xc000baee10 0xc0002ec770 0xc0002ec850 0xc0008a9590 0xc00653efc0 0xc0002ec8c0 map[0x2f67040:0xc002a07020 0x2f67580:0xc002a07260 0x2f68000:0xc002a06f60 0x2f682a0:0xc002a06fc0 0x2f687e0:0xc002a07080 0x2f68a80:0xc002a070e0 0x2f68d20:0xc002a07140 0x2f69500:0xc002a072c0 0x2f697a0:0xc002a07320 0x2f93d40:0xc0009f9bc0 0x2f940e0:0xc0009f9c20 0x2f94bc0:0xc000c86120 0x2f96520:0xc001bc4000 0x2f97000:0xc002a06f00 0x2f97e80:0xc0009f9b60] 0xc000168690 0} I0331 03:55:34.798772 3792 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c84070 0xc0002ec4d0 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0002ec690 0xc0002ec700 0xc000baee10 0xc0002ec770 0xc0002ec850 0xc0008a9590 0xc00653efc0 0xc0002ec8c0 map[0x2f67040:0xc002a07020 0x2f67580:0xc002a07260 0x2f68000:0xc002a06f60 0x2f682a0:0xc002a06fc0 0x2f687e0:0xc002a07080 0x2f68a80:0xc002a070e0 0x2f68d20:0xc002a07140 0x2f69500:0xc002a072c0 0x2f697a0:0xc002a07320 0x2f93d40:0xc0009f9bc0 0x2f940e0:0xc0009f9c20 0x2f94bc0:0xc000c86120 0x2f96520:0xc001bc4000 0x2f97000:0xc002a06f00 0x2f97e80:0xc0009f9b60] 0xc000168690 0} I0331 03:55:34.798811 3792 gateway.go:519] Reconciling gateway with updates I0331 03:55:34.804477 3792 gateway.go:326] Spawning Conntrack Rule Check Thread I0331 03:55:34.804562 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:55:34.804592 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:55:34.804595 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:55:34.804602 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:55:34.804619 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:55:34.804618 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:55:34.804618 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:55:34.804627 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/downloads I0331 03:55:34.804639 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:55:34.804648 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/downloads I0331 03:55:34.804670 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:55:34.804604 3792 default_node_network_controller.go:943] Gateway and management port readiness took 78.570962ms I0331 03:55:34.804684 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:55:34.804686 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:55:34.804693 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:55:34.804694 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0331 03:55:34.804696 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:55:34.804707 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:55:34.804710 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:55:34.804714 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0331 03:55:34.804719 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:55:34.804721 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:55:34.804720 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:55:34.804725 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0331 03:55:34.804725 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:55:34.804728 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:55:34.804722 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:55:34.804735 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:55:34.804735 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:55:34.804742 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:55:34.804746 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:55:34.804700 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:55:34.804757 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:55:34.804706 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:55:34.804564 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:55:34.804742 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:55:34.804795 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:55:34.804802 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:55:34.804807 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:55:34.804812 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:55:34.804821 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:55:34.804830 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:55:34.804837 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:55:34.804843 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:55:34.804834 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/console I0331 03:55:34.804852 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:55:34.804851 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:55:34.804872 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:55:34.804843 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:55:34.804889 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:55:34.804823 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:55:34.804710 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/downloads after 0 failed attempt(s) I0331 03:55:34.804930 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:55:34.804937 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:55:34.804943 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:55:34.804949 3792 port_claim.go:71] Svc openshift-ingress/router-default: port 30231 is already open, no action needed I0331 03:55:34.804952 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:55:34.804955 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:55:34.804958 3792 port_claim.go:71] Svc openshift-ingress/router-default: port 32345 is already open, no action needed I0331 03:55:34.804964 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:55:34.804975 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:55:34.804991 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:55:34.804978 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:55:34.805013 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:55:34.805016 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:55:34.805027 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:55:34.805029 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:55:34.805035 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:55:34.805037 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:55:34.805043 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:55:34.805051 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:55:34.805060 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:55:34.805067 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:55:34.805042 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:55:34.805078 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:55:34.805081 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:55:34.805084 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:55:34.805091 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:55:34.805062 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:55:34.805109 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:55:34.805113 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:55:34.805125 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:55:34.805129 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:55:34.805130 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:55:34.805134 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:55:34.805157 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:55:34.805162 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:55:34.805139 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:55:34.804671 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:55:34.805174 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:55:34.805176 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:55:34.805173 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:55:34.805188 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:55:34.805190 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:55:34.805192 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:55:34.805197 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:55:34.805205 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:55:34.805234 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:55:34.805245 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:55:34.805247 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:55:34.805253 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:55:34.805258 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:55:34.805262 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:55:34.805293 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:55:34.804855 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/console I0331 03:55:34.805305 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:55:34.804694 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:55:34.805320 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:55:34.804727 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:55:34.805310 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:55:34.805248 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:55:34.805488 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:55:34.805495 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:55:34.806252 3792 node_ip_handler_linux.go:160] Node IP manager is running W0331 03:55:34.840277 3792 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:55:34.840505 3792 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:55:34.840686 3792 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0331 03:55:34.861376 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:55:34.861411 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:55:34.861424 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:55:34.861439 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:55:34.861450 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:55:34.861459 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:55:34.861470 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:55:34.861480 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:55:34.861490 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:55:34.861500 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:55:34.861510 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:55:34.861518 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:55:34.861529 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:55:34.861537 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:55:34.861543 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:55:34.861548 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:55:34.901170 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:55:34.901198 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:55:34.901206 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook after 0 failed attempt(s) I0331 03:55:34.901216 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:55:34.901224 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:55:34.901243 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:55:34.901267 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/console after 0 failed attempt(s) I0331 03:55:34.901286 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:55:34.901294 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:55:34.901300 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:55:34.901307 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:55:34.901325 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:55:34.901360 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:55:34.901368 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:55:34.901374 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:55:34.940478 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:55:34.940488 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:55:34.940506 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) W0331 03:55:35.045975 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0331 03:55:36.615695 3792 default_node_network_controller.go:1364] Starting node controller node sync W0331 03:55:36.615757 3792 egressip_healthcheck.go:76] Health checking using insecure connection I0331 03:55:36.615820 3792 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0331 03:55:36.627251 3792 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.537858ms I0331 03:55:36.627350 3792 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-188.ec2.internal", IP: 10.0.2.188 to PMTUD blocking rules I0331 03:55:36.627353 3792 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-60.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:55:36.627360 3792 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-188.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:55:36.627364 3792 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-60.ec2.internal", IP: 10.0.0.60 to PMTUD blocking rules I0331 03:55:36.637913 3792 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0331 03:55:36.638212 3792 egressservice_node.go:93] Setting up event handlers for Egress Services I0331 03:55:36.638320 3792 egressservice_node.go:183] Starting Egress Services Controller I0331 03:55:36.638344 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:55:36.638356 3792 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:55:36.638362 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:55:36.638366 3792 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:55:36.638371 3792 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:55:36.638375 3792 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:55:36.638380 3792 egressservice_node.go:197] Repairing Egress Services I0331 03:55:36.647944 3792 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0331 03:55:36.647956 3792 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:55:36.648207 3792 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0331 03:55:36.648216 3792 link_network_manager.go:117] Link manager is running I0331 03:55:36.648224 3792 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0331 03:55:36.648310 3792 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0331 03:55:36.648861 3792 vrf_manager.go:128] VRF manager is running I0331 03:55:36.648949 3792 node_controller_manager.go:437] Removing flows to drop GARP I0331 03:55:36.648959 3792 gateway.go:519] Reconciling gateway with updates I0331 03:55:36.670038 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:55:36.670044 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:55:36.670059 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:55:36.670061 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:55:36.670063 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:55:36.670073 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:55:36.670075 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:55:36.670079 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:55:36.670087 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:55:36.670094 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:55:36.670111 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:55:36.670112 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:55:36.670124 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:55:36.670131 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:55:36.670134 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:55:36.670135 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:55:36.670155 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:55:36.670138 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:55:36.670160 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:55:36.670166 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:55:36.670175 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:55:36.670182 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:55:36.670193 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:55:36.670200 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:55:36.670055 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:55:36.670210 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:55:36.670127 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:55:36.670218 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:55:36.670220 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:55:36.670217 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:55:36.670227 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:55:36.670227 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:55:36.670237 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:55:36.670241 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:55:36.670249 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:55:36.670250 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:55:36.670249 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:55:36.670253 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:55:36.670256 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:55:36.670263 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:55:36.670265 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:55:36.670270 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:55:36.670270 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:55:36.670272 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:55:36.670268 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:55:36.670279 3792 port_claim.go:71] Svc openshift-ingress/router-default: port 30231 is already open, no action needed I0331 03:55:36.670283 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:55:36.670213 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:55:36.670287 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:55:36.670292 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0331 03:55:36.670296 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:55:36.670236 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:55:36.670303 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:55:36.670309 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:55:36.670308 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:55:36.670315 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:55:36.670316 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:55:36.670318 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:55:36.670323 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:55:36.670327 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:55:36.670334 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:55:36.670174 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:55:36.670293 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:55:36.670320 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:55:36.670369 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:55:36.670241 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:55:36.670383 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:55:36.670251 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:55:36.670196 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:55:36.670222 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:55:36.670428 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:55:36.670444 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:55:36.670445 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:55:36.670449 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:55:36.670455 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:55:36.670459 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:55:36.670472 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:55:36.670484 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:55:36.670490 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:55:36.670490 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:55:36.670490 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:55:36.670499 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:55:36.670499 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:55:36.670514 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:55:36.670202 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:55:36.670509 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:55:36.670475 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:55:36.670530 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:55:36.670457 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:55:36.670499 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:55:36.670253 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/downloads I0331 03:55:36.670564 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/downloads I0331 03:55:36.670572 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:55:36.670236 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:55:36.670242 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:55:36.670630 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:55:36.670256 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:55:36.670630 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:55:36.670351 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:55:36.670686 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:55:36.670696 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:55:36.670417 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:55:36.670702 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:55:36.670302 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0331 03:55:36.670707 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:55:36.670711 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0331 03:55:36.670711 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:55:36.670455 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:55:36.670722 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:55:36.670726 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:55:36.670228 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:55:36.670455 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/console I0331 03:55:36.670756 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/console I0331 03:55:36.670448 3792 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:55:36.670811 3792 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:55:36.670289 3792 port_claim.go:71] Svc openshift-ingress/router-default: port 32345 is already open, no action needed I0331 03:55:36.707346 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:55:36.707362 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:55:36.707369 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:55:36.707375 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook after 0 failed attempt(s) I0331 03:55:36.707380 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:55:36.707385 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:55:36.707390 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:55:36.707396 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743799 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:55:36.743827 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:55:36.743834 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:55:36.743841 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743847 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743855 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743863 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:55:36.743868 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743874 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:55:36.743880 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:55:36.743885 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:55:36.743893 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:55:36.743899 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743904 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/downloads after 0 failed attempt(s) I0331 03:55:36.743910 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:55:36.743915 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:55:36.743922 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:55:36.743927 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:55:36.743933 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:55:36.743940 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/console after 0 failed attempt(s) I0331 03:55:36.743945 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:55:36.780291 3792 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) W0331 03:55:37.060672 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0331 03:55:38.521929 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 03:55:38.521956 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 36.062µs I0331 03:55:38.550666 3792 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-x9g85 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.550868 3792 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85 I0331 03:55:38.555907 3792 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-m8dgn] creating logical port openshift-console_downloads-56d45c498b-m8dgn for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.556003 3792 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-m8dgn I0331 03:55:38.556183 3792 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c8d4556db-4jls2] creating logical port openshift-image-registry_image-registry-7c8d4556db-4jls2 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.556270 3792 kube.go:256] Updating pod openshift-image-registry/image-registry-7c8d4556db-4jls2 I0331 03:55:38.569386 3792 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85] pod update took 18.572787ms I0331 03:55:38.569404 3792 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85] addLogicalPort annotation time took 18.597212ms I0331 03:55:38.570639 3792 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85] addLogicalPort took 19.983264ms, libovsdb time 850.767µs I0331 03:55:38.572463 3792 pod.go:62] [openshift-console/downloads-56d45c498b-m8dgn] pod update took 16.497405ms I0331 03:55:38.572478 3792 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-m8dgn] addLogicalPort annotation time took 16.516134ms I0331 03:55:38.573537 3792 pods.go:271] [openshift-console/downloads-56d45c498b-m8dgn] addLogicalPort took 17.64344ms, libovsdb time 794.548µs I0331 03:55:38.573588 3792 base_network_controller_pods.go:487] [default/openshift-console/console-5dfd8574fd-6hg6d] creating logical port openshift-console_console-5dfd8574fd-6hg6d for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.573666 3792 kube.go:256] Updating pod openshift-console/console-5dfd8574fd-6hg6d I0331 03:55:38.575608 3792 pod.go:62] [openshift-image-registry/image-registry-7c8d4556db-4jls2] pod update took 19.370773ms I0331 03:55:38.575625 3792 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7c8d4556db-4jls2] addLogicalPort annotation time took 19.392534ms I0331 03:55:38.576549 3792 pods.go:271] [openshift-image-registry/image-registry-7c8d4556db-4jls2] addLogicalPort took 20.378346ms, libovsdb time 708.237µs I0331 03:55:38.578626 3792 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-ffc7ccb6f-j8q9k] creating logical port openshift-ingress_router-default-ffc7ccb6f-j8q9k for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.578682 3792 kube.go:256] Updating pod openshift-ingress/router-default-ffc7ccb6f-j8q9k I0331 03:55:38.580960 3792 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-msfpp] creating logical port openshift-ingress-canary_ingress-canary-msfpp for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.581041 3792 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-msfpp I0331 03:55:38.590534 3792 pod.go:62] [openshift-console/console-5dfd8574fd-6hg6d] pod update took 16.858862ms I0331 03:55:38.590554 3792 base_network_controller_pods.go:937] [default/openshift-console/console-5dfd8574fd-6hg6d] addLogicalPort annotation time took 16.913275ms I0331 03:55:38.591576 3792 pods.go:271] [openshift-console/console-5dfd8574fd-6hg6d] addLogicalPort took 17.997231ms, libovsdb time 751.761µs I0331 03:55:38.595552 3792 pod.go:62] [openshift-ingress-canary/ingress-canary-msfpp] pod update took 14.526951ms I0331 03:55:38.595575 3792 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-msfpp] addLogicalPort annotation time took 14.549716ms I0331 03:55:38.596613 3792 pods.go:271] [openshift-ingress-canary/ingress-canary-msfpp] addLogicalPort took 15.663951ms, libovsdb time 751.487µs I0331 03:55:38.597582 3792 pod.go:62] [openshift-ingress/router-default-ffc7ccb6f-j8q9k] pod update took 18.916819ms I0331 03:55:38.597601 3792 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-ffc7ccb6f-j8q9k] addLogicalPort annotation time took 18.938119ms I0331 03:55:38.597979 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-60.ec2.internal I0331 03:55:38.598012 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-60.ec2.internal: 37.764µs I0331 03:55:38.598577 3792 pods.go:271] [openshift-ingress/router-default-ffc7ccb6f-j8q9k] addLogicalPort took 19.959158ms, libovsdb time 740.053µs I0331 03:55:38.673037 3792 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-knqb2] creating logical port openshift-dns_dns-default-knqb2 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.673157 3792 kube.go:256] Updating pod openshift-dns/dns-default-knqb2 I0331 03:55:38.682802 3792 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-84lb6] creating logical port openshift-insights_insights-runtime-extractor-84lb6 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:38.682954 3792 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-84lb6 I0331 03:55:38.693910 3792 pod.go:62] [openshift-dns/dns-default-knqb2] pod update took 20.799551ms I0331 03:55:38.693930 3792 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-knqb2] addLogicalPort annotation time took 20.832029ms I0331 03:55:38.695106 3792 pods.go:271] [openshift-dns/dns-default-knqb2] addLogicalPort took 22.086039ms, libovsdb time 853.872µs I0331 03:55:38.705396 3792 pod.go:62] [openshift-insights/insights-runtime-extractor-84lb6] pod update took 22.505453ms I0331 03:55:38.705416 3792 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-84lb6] addLogicalPort annotation time took 22.528025ms I0331 03:55:38.706474 3792 pods.go:271] [openshift-insights/insights-runtime-extractor-84lb6] addLogicalPort took 23.684984ms, libovsdb time 786.301µs I0331 03:55:39.062467 3792 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85 96c1ea02cd175e58386a1dc67b586dc6a9ba661115e1411f5c6332069dfca9ef network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85 96c1ea02cd175e58386a1dc67b586dc6a9ba661115e1411f5c6332069dfca9ef network default NAD default] I0331 03:55:39.097943 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-x9g85, hostIfaceName: 96c1ea02cd175e5, network: default, NAD default, SandboxID: "96c1ea02cd175e58386a1dc67b586dc6a9ba661115e1411f5c6332069dfca9ef", PCI device ID: , UID: "a0ff818b-a452-4ea2-819b-ee9ff850d596", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0331 03:55:39.099495 3792 cni.go:403] [openshift-console/downloads-56d45c498b-m8dgn 1743d9fa3aababb8917529678bb4dd5cbc4b8c32b15bc245928e9bb041c6f308 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-m8dgn 1743d9fa3aababb8917529678bb4dd5cbc4b8c32b15bc245928e9bb041c6f308 network default NAD default] I0331 03:55:39.109014 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-m8dgn, hostIfaceName: 1743d9fa3aababb, network: default, NAD default, SandboxID: "1743d9fa3aababb8917529678bb4dd5cbc4b8c32b15bc245928e9bb041c6f308", PCI device ID: , UID: "9a82c79a-eca2-422f-ac3b-2558a8e07ea3", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0331 03:55:39.140485 3792 cni.go:403] [openshift-ingress-canary/ingress-canary-msfpp ff73d1b62e1a295bd003148a2421c7cb04b63892e9d1d6a0ee85cc2c5c1a9356 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-msfpp ff73d1b62e1a295bd003148a2421c7cb04b63892e9d1d6a0ee85cc2c5c1a9356 network default NAD default] I0331 03:55:39.149629 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-msfpp, hostIfaceName: ff73d1b62e1a295, network: default, NAD default, SandboxID: "ff73d1b62e1a295bd003148a2421c7cb04b63892e9d1d6a0ee85cc2c5c1a9356", PCI device ID: , UID: "c561fcc8-02a4-472b-adbd-4aa463211bc6", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0331 03:55:39.153999 3792 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85 96c1ea02cd175e58386a1dc67b586dc6a9ba661115e1411f5c6332069dfca9ef network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-x9g85 96c1ea02cd175e58386a1dc67b586dc6a9ba661115e1411f5c6332069dfca9ef network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:98:30:d9:b2:29\",\"name\":\"96c1ea02cd175e5\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0717c53a-80b9-494d-8b97-3fed46d596c2\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.156353 3792 cni.go:403] [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default] ADD starting CNI request [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default] I0331 03:55:39.159220 3792 cni.go:424] [openshift-console/downloads-56d45c498b-m8dgn 1743d9fa3aababb8917529678bb4dd5cbc4b8c32b15bc245928e9bb041c6f308 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-m8dgn 1743d9fa3aababb8917529678bb4dd5cbc4b8c32b15bc245928e9bb041c6f308 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:47:b6:41:8b:e5\",\"name\":\"1743d9fa3aababb\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80e8db32-118d-44f9-b336-89d1c5f0b418\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.173337 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5dfd8574fd-6hg6d, hostIfaceName: e546a0695d40185, network: default, NAD default, SandboxID: "e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e", PCI device ID: , UID: "7a2322da-446f-428a-922f-ce255a2b2075", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0331 03:55:39.180823 3792 cni.go:403] [openshift-image-registry/image-registry-7c8d4556db-4jls2 22a059443fcc0ab6cc7b7c7de6c8d35bba25a7176a48fb4242b23181d83e5cb9 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7c8d4556db-4jls2 22a059443fcc0ab6cc7b7c7de6c8d35bba25a7176a48fb4242b23181d83e5cb9 network default NAD default] I0331 03:55:39.192029 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7c8d4556db-4jls2, hostIfaceName: 22a059443fcc0ab, network: default, NAD default, SandboxID: "22a059443fcc0ab6cc7b7c7de6c8d35bba25a7176a48fb4242b23181d83e5cb9", PCI device ID: , UID: "dee5605a-669a-4d98-8d9e-39e537164e81", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0331 03:55:39.195844 3792 cni.go:403] [openshift-ingress/router-default-ffc7ccb6f-j8q9k 220cc306cfb9851f74c3615cea365582c449ea675a73d29a96f5772b318da60e network default NAD default] ADD starting CNI request [openshift-ingress/router-default-ffc7ccb6f-j8q9k 220cc306cfb9851f74c3615cea365582c449ea675a73d29a96f5772b318da60e network default NAD default] I0331 03:55:39.200485 3792 cni.go:424] [openshift-ingress-canary/ingress-canary-msfpp ff73d1b62e1a295bd003148a2421c7cb04b63892e9d1d6a0ee85cc2c5c1a9356 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-msfpp ff73d1b62e1a295bd003148a2421c7cb04b63892e9d1d6a0ee85cc2c5c1a9356 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:6b:a1:6a:56:f4\",\"name\":\"ff73d1b62e1a295\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0ec5e0b-ba70-4573-b706-99ba4d9e12cf\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.214668 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-ffc7ccb6f-j8q9k, hostIfaceName: 220cc306cfb9851, network: default, NAD default, SandboxID: "220cc306cfb9851f74c3615cea365582c449ea675a73d29a96f5772b318da60e", PCI device ID: , UID: "26a5ca55-d82f-4040-96a6-037b7b2a6296", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0331 03:55:39.214724 3792 cni.go:403] [openshift-insights/insights-runtime-extractor-84lb6 68e4412e000ace3fd1051dc2c1cf8b38dcc7862f0165a38b0c015ec2f7d61ea1 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-84lb6 68e4412e000ace3fd1051dc2c1cf8b38dcc7862f0165a38b0c015ec2f7d61ea1 network default NAD default] I0331 03:55:39.215931 3792 cni.go:403] [openshift-dns/dns-default-knqb2 b991ff64d8dec81445fceebf42b5686b14b7316de1ce0674eb3dc9a4796f17f5 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-knqb2 b991ff64d8dec81445fceebf42b5686b14b7316de1ce0674eb3dc9a4796f17f5 network default NAD default] I0331 03:55:39.231669 3792 cni.go:424] [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default] ADD finished CNI request [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:6b:78:f9:f7:d1\",\"name\":\"e546a0695d40185\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/914c7303-809a-4fe3-856e-bed17b3aef36\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.232551 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-84lb6, hostIfaceName: 68e4412e000ace3, network: default, NAD default, SandboxID: "68e4412e000ace3fd1051dc2c1cf8b38dcc7862f0165a38b0c015ec2f7d61ea1", PCI device ID: , UID: "4606f67c-fe46-4c92-82ed-24580b60055d", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0331 03:55:39.246270 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-knqb2, hostIfaceName: b991ff64d8dec81, network: default, NAD default, SandboxID: "b991ff64d8dec81445fceebf42b5686b14b7316de1ce0674eb3dc9a4796f17f5", PCI device ID: , UID: "d4eff620-133c-4967-8e0f-910d9e59c8c7", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0331 03:55:39.253437 3792 cni.go:424] [openshift-image-registry/image-registry-7c8d4556db-4jls2 22a059443fcc0ab6cc7b7c7de6c8d35bba25a7176a48fb4242b23181d83e5cb9 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7c8d4556db-4jls2 22a059443fcc0ab6cc7b7c7de6c8d35bba25a7176a48fb4242b23181d83e5cb9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:d1:f7:80:69:aa\",\"name\":\"22a059443fcc0ab\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90ef8d20-1dd6-4d00-8bb7-45db40ea2749\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.264536 3792 cni.go:424] [openshift-ingress/router-default-ffc7ccb6f-j8q9k 220cc306cfb9851f74c3615cea365582c449ea675a73d29a96f5772b318da60e network default NAD default] ADD finished CNI request [openshift-ingress/router-default-ffc7ccb6f-j8q9k 220cc306cfb9851f74c3615cea365582c449ea675a73d29a96f5772b318da60e network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:db:60:84:76:08\",\"name\":\"220cc306cfb9851\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e0ec78f1-8911-49f7-ab03-67db81886853\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.278617 3792 cni.go:424] [openshift-insights/insights-runtime-extractor-84lb6 68e4412e000ace3fd1051dc2c1cf8b38dcc7862f0165a38b0c015ec2f7d61ea1 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-84lb6 68e4412e000ace3fd1051dc2c1cf8b38dcc7862f0165a38b0c015ec2f7d61ea1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:6f:90:85:79:1e\",\"name\":\"68e4412e000ace3\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/86122195-073b-4e4f-a29c-109e30a9b963\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:39.287634 3792 cni.go:424] [openshift-dns/dns-default-knqb2 b991ff64d8dec81445fceebf42b5686b14b7316de1ce0674eb3dc9a4796f17f5 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-knqb2 b991ff64d8dec81445fceebf42b5686b14b7316de1ce0674eb3dc9a4796f17f5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:af:b7:a6:d4:7e\",\"name\":\"b991ff64d8dec81\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5b99162c-0199-4c9c-bc6a-a0a447ed7edc\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 03:55:39.357743 3792 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:40.395806 3792 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:40.431811 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0331 03:55:40.431899 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0331 03:55:41.368031 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0331 03:55:41.368059 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0331 03:55:41.461114 3792 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0331 03:55:41.475828 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:55:41.475852 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:55:41.475861 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:55:47.949393 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:47.949426 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:55:49.113856 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0331 03:55:49.406792 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 03:55:51.974510 3792 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:51.974615 3792 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0331 03:55:52.006312 3792 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 31.737787ms I0331 03:55:52.006371 3792 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 31.770445ms I0331 03:55:52.007785 3792 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 33.29091ms, libovsdb time 986.802µs W0331 03:55:52.370966 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:52.370987 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:52.370992 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:52.371006 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:52.371014 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:52.371021 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default I0331 03:55:52.958638 3792 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-667999dd5-2drkm] creating logical port openshift-monitoring_thanos-querier-667999dd5-2drkm for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:52.958784 3792 kube.go:256] Updating pod openshift-monitoring/thanos-querier-667999dd5-2drkm I0331 03:55:52.977068 3792 pod.go:62] [openshift-monitoring/thanos-querier-667999dd5-2drkm] pod update took 18.324191ms I0331 03:55:52.977090 3792 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-667999dd5-2drkm] addLogicalPort annotation time took 18.349157ms I0331 03:55:52.978252 3792 pods.go:271] [openshift-monitoring/thanos-querier-667999dd5-2drkm] addLogicalPort took 19.635399ms, libovsdb time 850.658µs I0331 03:55:53.979341 3792 cni.go:403] [openshift-monitoring/thanos-querier-667999dd5-2drkm b38973359c4939659fb1a37c30499a3d65dddffb290808ed2e0766b653e90019 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-667999dd5-2drkm b38973359c4939659fb1a37c30499a3d65dddffb290808ed2e0766b653e90019 network default NAD default] I0331 03:55:53.979745 3792 cni.go:403] [openshift-monitoring/alertmanager-main-1 d1e2aa45f41ee3f78b01774dee231431234e45f8ce54412b5cbe3c1b4167b2be network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 d1e2aa45f41ee3f78b01774dee231431234e45f8ce54412b5cbe3c1b4167b2be network default NAD default] I0331 03:55:53.990971 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: d1e2aa45f41ee3f, network: default, NAD default, SandboxID: "d1e2aa45f41ee3f78b01774dee231431234e45f8ce54412b5cbe3c1b4167b2be", PCI device ID: , UID: "fab8ca4e-2758-42ff-9abd-f20c1f650ade", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0331 03:55:53.991319 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-667999dd5-2drkm, hostIfaceName: b38973359c49396, network: default, NAD default, SandboxID: "b38973359c4939659fb1a37c30499a3d65dddffb290808ed2e0766b653e90019", PCI device ID: , UID: "df3899f7-502f-4375-8571-f3decbde73b3", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0331 03:55:54.032926 3792 cni.go:424] [openshift-monitoring/thanos-querier-667999dd5-2drkm b38973359c4939659fb1a37c30499a3d65dddffb290808ed2e0766b653e90019 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-667999dd5-2drkm b38973359c4939659fb1a37c30499a3d65dddffb290808ed2e0766b653e90019 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:c2:4f:72:19:0a\",\"name\":\"b38973359c49396\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b09410ab-4d6e-463a-81cf-d5ca685d1071\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:54.047575 3792 cni.go:424] [openshift-monitoring/alertmanager-main-1 d1e2aa45f41ee3f78b01774dee231431234e45f8ce54412b5cbe3c1b4167b2be network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 d1e2aa45f41ee3f78b01774dee231431234e45f8ce54412b5cbe3c1b4167b2be network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:db:fb:a3:78:69\",\"name\":\"d1e2aa45f41ee3f\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c852a7c-5365-4a5f-aa65-6008155ea7f9\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 03:55:54.389301 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:54.389322 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:55:54.389328 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0331 03:55:55.367591 3792 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-5c9476ccdc-p76rm] creating logical port openshift-monitoring_metrics-server-5c9476ccdc-p76rm for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:55.367845 3792 kube.go:256] Updating pod openshift-monitoring/metrics-server-5c9476ccdc-p76rm I0331 03:55:55.386287 3792 pod.go:62] [openshift-monitoring/metrics-server-5c9476ccdc-p76rm] pod update took 18.48856ms I0331 03:55:55.386317 3792 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-5c9476ccdc-p76rm] addLogicalPort annotation time took 18.522894ms I0331 03:55:55.387937 3792 pods.go:271] [openshift-monitoring/metrics-server-5c9476ccdc-p76rm] addLogicalPort took 20.367237ms, libovsdb time 1.162218ms W0331 03:55:55.473466 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0331 03:55:55.607965 3792 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-rhjs7 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:55.608065 3792 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7 I0331 03:55:55.629649 3792 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7] pod update took 21.607056ms I0331 03:55:55.629674 3792 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7] addLogicalPort annotation time took 21.636523ms I0331 03:55:55.630879 3792 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7] addLogicalPort took 22.928559ms, libovsdb time 825.828µs W0331 03:55:56.466481 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:56.466501 3792 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:56.466509 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0331 03:55:56.513862 3792 cni.go:403] [openshift-monitoring/metrics-server-5c9476ccdc-p76rm 57a8b746734136cbb010653eaed0778241724321c21fa10a9aac5afbbc0db471 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-5c9476ccdc-p76rm 57a8b746734136cbb010653eaed0778241724321c21fa10a9aac5afbbc0db471 network default NAD default] I0331 03:55:56.524802 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-5c9476ccdc-p76rm, hostIfaceName: 57a8b746734136c, network: default, NAD default, SandboxID: "57a8b746734136cbb010653eaed0778241724321c21fa10a9aac5afbbc0db471", PCI device ID: , UID: "764b9807-593e-42e1-96db-169c155cd7a0", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0331 03:55:56.579010 3792 cni.go:424] [openshift-monitoring/metrics-server-5c9476ccdc-p76rm 57a8b746734136cbb010653eaed0778241724321c21fa10a9aac5afbbc0db471 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-5c9476ccdc-p76rm 57a8b746734136cbb010653eaed0778241724321c21fa10a9aac5afbbc0db471 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:06:74:2b:64:e8\",\"name\":\"57a8b746734136c\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79703df1-7389-494d-9a1b-a0b5ec5519a9\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:56.622529 3792 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7 8d8deb8c14e5aa6076b7673f95de996d45eca0f93f84340524d9ee355985870c network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7 8d8deb8c14e5aa6076b7673f95de996d45eca0f93f84340524d9ee355985870c network default NAD default] I0331 03:55:56.631667 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-rhjs7, hostIfaceName: 8d8deb8c14e5aa6, network: default, NAD default, SandboxID: "8d8deb8c14e5aa6076b7673f95de996d45eca0f93f84340524d9ee355985870c", PCI device ID: , UID: "638f1912-3943-4cce-be0b-f88af3036a8d", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0331 03:55:56.679430 3792 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7 8d8deb8c14e5aa6076b7673f95de996d45eca0f93f84340524d9ee355985870c network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-rhjs7 8d8deb8c14e5aa6076b7673f95de996d45eca0f93f84340524d9ee355985870c network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:51:de:c9:e5:7c\",\"name\":\"8d8deb8c14e5aa6\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0130f689-e2e1-495f-b591-d4e3989af1fa\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:55:57.234070 3792 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:55:57.234230 3792 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0331 03:55:57.264757 3792 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 30.562996ms I0331 03:55:57.264788 3792 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 30.600917ms I0331 03:55:57.266137 3792 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 32.078878ms, libovsdb time 978.488µs W0331 03:55:57.506741 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:57.506766 3792 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:55:57.506772 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0331 03:55:59.119862 3792 cni.go:403] [openshift-monitoring/prometheus-k8s-1 c8ef421b92614eebdf646c8eb3a7a7bb7e0270fdc9d87c6728a28613d3266ea5 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 c8ef421b92614eebdf646c8eb3a7a7bb7e0270fdc9d87c6728a28613d3266ea5 network default NAD default] I0331 03:55:59.129883 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: c8ef421b92614ee, network: default, NAD default, SandboxID: "c8ef421b92614eebdf646c8eb3a7a7bb7e0270fdc9d87c6728a28613d3266ea5", PCI device ID: , UID: "fef75667-b47a-4db7-9c2a-5f6c5d5bb84a", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0331 03:55:59.184137 3792 cni.go:424] [openshift-monitoring/prometheus-k8s-1 c8ef421b92614eebdf646c8eb3a7a7bb7e0270fdc9d87c6728a28613d3266ea5 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 c8ef421b92614eebdf646c8eb3a7a7bb7e0270fdc9d87c6728a28613d3266ea5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:f6:e9:9a:59:82\",\"name\":\"c8ef421b92614ee\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6f921760-d638-44ca-a125-3cffe9d5e168\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 03:55:59.449955 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0331 03:55:59.472045 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0331 03:55:59.517270 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:55:59.517297 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:55:59.536851 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0331 03:56:00.329036 3792 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.329695 3792 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.329720 3792 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.329730 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0331 03:56:00.329754 3792 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:56:00.329762 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0331 03:56:00.405240 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:56:00.405273 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:56:00.461717 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:56:00.461741 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:56:00.493126 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.493168 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.493178 3792 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.493183 3792 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.501607 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:56:00.556950 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:00.556974 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:00.634175 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.634194 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.634204 3792 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:00.634211 3792 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:56:01.358755 3792 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:01.358781 3792 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:01.358801 3792 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:01.358811 3792 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xnzjp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0331 03:56:01.374965 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:56:01.578130 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578173 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578184 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578192 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578212 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578219 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578225 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:01.578232 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:56:04.792424 3792 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0331 03:56:04.800620 3792 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0331 03:56:05.215887 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0331 03:56:06.324549 3792 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-sjjqn for pod on switch ip-10-0-1-138.ec2.internal I0331 03:56:06.324654 3792 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn I0331 03:56:06.338608 3792 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn] pod update took 13.987842ms I0331 03:56:06.338627 3792 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn] addLogicalPort annotation time took 14.009916ms I0331 03:56:06.339801 3792 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn] addLogicalPort took 15.268862ms, libovsdb time 839.692µs W0331 03:56:06.495043 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:06.495070 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:06.495078 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:56:06.495086 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:56:06.743605 3792 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn 55ecaf2b8ee22613d694543766dfabcad50537713116a02989bf94fdec2b21b4 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn 55ecaf2b8ee22613d694543766dfabcad50537713116a02989bf94fdec2b21b4 network default NAD default] I0331 03:56:06.752170 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-sjjqn, hostIfaceName: 55ecaf2b8ee2261, network: default, NAD default, SandboxID: "55ecaf2b8ee22613d694543766dfabcad50537713116a02989bf94fdec2b21b4", PCI device ID: , UID: "791e28e3-ff23-4e4c-b442-1c5593b05ff7", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0331 03:56:06.794354 3792 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn 55ecaf2b8ee22613d694543766dfabcad50537713116a02989bf94fdec2b21b4 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-sjjqn 55ecaf2b8ee22613d694543766dfabcad50537713116a02989bf94fdec2b21b4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:70:c7:53:ad:39\",\"name\":\"55ecaf2b8ee2261\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9edcd65-cb49-4f9c-ad12-9fd90badbc06\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 03:56:10.249392 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0331 03:56:10.279185 3792 cni.go:403] [openshift-multus/network-metrics-daemon-nwlvp f9c88eec5298f1dae704af4a0df9bf363fce47353cd71746d558abdb4482ebbc network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-nwlvp f9c88eec5298f1dae704af4a0df9bf363fce47353cd71746d558abdb4482ebbc network default NAD default] I0331 03:56:10.288793 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-nwlvp, hostIfaceName: f9c88eec5298f1d, network: default, NAD default, SandboxID: "f9c88eec5298f1dae704af4a0df9bf363fce47353cd71746d558abdb4482ebbc", PCI device ID: , UID: "144680e7-aea2-43cc-b39b-4420c1d434d7", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0331 03:56:10.309111 3792 cni.go:403] [openshift-network-diagnostics/network-check-target-st94r b6fec2ee9f2ec445deea3b9c799feb224ce7e296da50a41fb11d33e5521bce0c network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-st94r b6fec2ee9f2ec445deea3b9c799feb224ce7e296da50a41fb11d33e5521bce0c network default NAD default] I0331 03:56:10.318715 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-st94r, hostIfaceName: b6fec2ee9f2ec44, network: default, NAD default, SandboxID: "b6fec2ee9f2ec445deea3b9c799feb224ce7e296da50a41fb11d33e5521bce0c", PCI device ID: , UID: "8eb7a99c-6a5e-4723-a6d9-edfabcbce1cf", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0331 03:56:10.342982 3792 cni.go:424] [openshift-multus/network-metrics-daemon-nwlvp f9c88eec5298f1dae704af4a0df9bf363fce47353cd71746d558abdb4482ebbc network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-nwlvp f9c88eec5298f1dae704af4a0df9bf363fce47353cd71746d558abdb4482ebbc network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:d8:50:49:a5:f0\",\"name\":\"f9c88eec5298f1d\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9cd50f99-9d04-47eb-abeb-b82394e2c94f\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:56:10.363021 3792 cni.go:424] [openshift-network-diagnostics/network-check-target-st94r b6fec2ee9f2ec445deea3b9c799feb224ce7e296da50a41fb11d33e5521bce0c network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-st94r b6fec2ee9f2ec445deea3b9c799feb224ce7e296da50a41fb11d33e5521bce0c network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:cd:ed:cd:b0:e4\",\"name\":\"b6fec2ee9f2ec44\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/14b98e41-dba3-4283-9083-2daa9852910d\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 03:56:10.530977 3792 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0331 03:56:16.266880 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0331 03:56:20.808911 3792 cni.go:403] [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default] DEL starting CNI request [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default] I0331 03:56:20.880303 3792 cni.go:424] [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default] DEL finished CNI request [openshift-console/console-5dfd8574fd-6hg6d e546a0695d40185a4705b9100c8521c99027390a58de97769d785dcf51d1fc4e network default NAD default], result "{}", err I0331 03:56:20.944675 3792 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:20.944724 3792 pods.go:176] Deleting pod: openshift-console/console-5dfd8574fd-6hg6d I0331 03:56:20.946020 3792 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5dfd8574fd-6hg6d, ips: 10.130.0.8 I0331 03:56:20.946046 3792 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:20.953168 3792 base_network_controller_pods.go:487] [default/openshift-console/console-6877b44645-tchg7] creating logical port openshift-console_console-6877b44645-tchg7 for pod on switch ip-10-0-1-138.ec2.internal I0331 03:56:20.953299 3792 kube.go:256] Updating pod openshift-console/console-6877b44645-tchg7 I0331 03:56:20.967385 3792 pod.go:62] [openshift-console/console-6877b44645-tchg7] pod update took 14.142909ms I0331 03:56:20.967406 3792 base_network_controller_pods.go:937] [default/openshift-console/console-6877b44645-tchg7] addLogicalPort annotation time took 14.167945ms I0331 03:56:20.968560 3792 pods.go:271] [openshift-console/console-6877b44645-tchg7] addLogicalPort took 15.422061ms, libovsdb time 847.648µs I0331 03:56:21.329475 3792 cni.go:403] [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default] ADD starting CNI request [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default] I0331 03:56:21.338561 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6877b44645-tchg7, hostIfaceName: e0004382f3965a4, network: default, NAD default, SandboxID: "e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e", PCI device ID: , UID: "a36e6b73-5084-4618-a40f-2bc79c1b0b0f", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0331 03:56:21.381000 3792 cni.go:424] [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default] ADD finished CNI request [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:7a:f5:59:b6:ab\",\"name\":\"e0004382f3965a4\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f60c45e4-8061-4ad0-9059-686b3a73a837\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 03:56:21.485338 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:56:21.535348 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:56:21.573325 3792 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:21.573353 3792 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-6hg6d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:21.577890 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dfd8574fd-6hg6d I0331 03:56:21.577913 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dfd8574fd-6hg6d W0331 03:56:23.009629 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:56:23.009660 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:56:23.009668 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0331 03:56:25.789707 3792 obj_retry.go:464] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:25.790291 3792 obj_retry.go:464] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:26.315058 3792 obj_retry.go:458] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:26.315082 3792 obj_retry.go:458] Detected object openshift-image-registry/image-registry-75cc576b7d-pd4zw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:26.320572 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-75cc576b7d-pd4zw I0331 03:56:26.320599 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-75cc576b7d-pd4zw W0331 03:56:31.634554 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 03:56:31.895717 3792 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:31.896253 3792 obj_retry.go:464] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:32.338189 3792 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:32.338213 3792 obj_retry.go:458] Detected object openshift-console/console-5dfd8574fd-rf5pp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:32.345463 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dfd8574fd-rf5pp I0331 03:56:32.345489 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dfd8574fd-rf5pp W0331 03:56:35.519638 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:56:35.519667 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0331 03:56:40.354989 3792 obj_retry.go:464] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:40.355542 3792 obj_retry.go:464] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:56:40.679069 3792 obj_retry.go:458] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:40.679097 3792 obj_retry.go:458] Detected object openshift-ingress/router-default-6fb4878899-lt98g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:56:40.681927 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-6fb4878899-lt98g I0331 03:56:40.681947 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-6fb4878899-lt98g W0331 03:56:45.550120 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:56:57.768597 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:57.768626 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:57.768640 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:57.768646 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:59.775217 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:56:59.775245 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:57:28.738658 3792 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-796968dcf4-xj7wl IP address from the namespace address-set, err: pod openshift-console/console-796968dcf4-xj7wl: no pod IPs found I0331 03:57:43.244968 3792 cni.go:403] [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default] DEL starting CNI request [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default] I0331 03:57:43.327919 3792 cni.go:424] [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default] DEL finished CNI request [openshift-console/console-6877b44645-tchg7 e0004382f3965a4933da2c58228f8482c72818b8b7e8b57da3b0d49379534c5e network default NAD default], result "{}", err I0331 03:57:43.375986 3792 obj_retry.go:464] Detected object openshift-console/console-6877b44645-tchg7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:43.376022 3792 pods.go:176] Deleting pod: openshift-console/console-6877b44645-tchg7 I0331 03:57:43.377524 3792 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6877b44645-tchg7, ips: 10.130.0.19 I0331 03:57:43.377562 3792 obj_retry.go:464] Detected object openshift-console/console-6877b44645-tchg7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:43.384868 3792 base_network_controller_pods.go:487] [default/openshift-console/console-54764455dc-jvcfl] creating logical port openshift-console_console-54764455dc-jvcfl for pod on switch ip-10-0-1-138.ec2.internal I0331 03:57:43.384948 3792 kube.go:256] Updating pod openshift-console/console-54764455dc-jvcfl I0331 03:57:43.398614 3792 pod.go:62] [openshift-console/console-54764455dc-jvcfl] pod update took 13.68807ms I0331 03:57:43.398643 3792 base_network_controller_pods.go:937] [default/openshift-console/console-54764455dc-jvcfl] addLogicalPort annotation time took 13.718354ms I0331 03:57:43.399927 3792 pods.go:271] [openshift-console/console-54764455dc-jvcfl] addLogicalPort took 15.078658ms, libovsdb time 882.17µs I0331 03:57:43.742185 3792 cni.go:403] [openshift-console/console-54764455dc-jvcfl 23d23e856e28e09aa84f645dd9a3a3e1b2f5f81c0df8c3f2c6ef7f7cb3f90e29 network default NAD default] ADD starting CNI request [openshift-console/console-54764455dc-jvcfl 23d23e856e28e09aa84f645dd9a3a3e1b2f5f81c0df8c3f2c6ef7f7cb3f90e29 network default NAD default] I0331 03:57:43.750618 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-54764455dc-jvcfl, hostIfaceName: 23d23e856e28e09, network: default, NAD default, SandboxID: "23d23e856e28e09aa84f645dd9a3a3e1b2f5f81c0df8c3f2c6ef7f7cb3f90e29", PCI device ID: , UID: "82dfb3f8-dbdf-4037-9a02-6f7bb2221acb", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0331 03:57:43.792293 3792 cni.go:424] [openshift-console/console-54764455dc-jvcfl 23d23e856e28e09aa84f645dd9a3a3e1b2f5f81c0df8c3f2c6ef7f7cb3f90e29 network default NAD default] ADD finished CNI request [openshift-console/console-54764455dc-jvcfl 23d23e856e28e09aa84f645dd9a3a3e1b2f5f81c0df8c3f2c6ef7f7cb3f90e29 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:7e:97:87:1f:fc\",\"name\":\"23d23e856e28e09\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d6e7821d-f882-4d0d-b72d-3d130fb250ea\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 03:57:43.888272 3792 obj_retry.go:458] Detected object openshift-console/console-6877b44645-tchg7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:43.888298 3792 obj_retry.go:458] Detected object openshift-console/console-6877b44645-tchg7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:43.895538 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6877b44645-tchg7 I0331 03:57:43.895561 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6877b44645-tchg7 I0331 03:57:53.915729 3792 obj_retry.go:464] Detected object openshift-console/console-6877b44645-xxckv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:57:53.916330 3792 obj_retry.go:464] Detected object openshift-console/console-6877b44645-xxckv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 03:57:53.943491 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 03:57:54.247050 3792 obj_retry.go:458] Detected object openshift-console/console-6877b44645-xxckv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:54.247076 3792 obj_retry.go:458] Detected object openshift-console/console-6877b44645-xxckv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:57:54.254047 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6877b44645-xxckv I0331 03:57:54.254073 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6877b44645-xxckv I0331 03:58:00.963821 3792 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0331 03:58:19.272132 3792 obj_retry.go:464] Detected object openshift-console/console-796968dcf4-zd56g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:58:19.272688 3792 obj_retry.go:464] Detected object openshift-console/console-796968dcf4-zd56g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:58:19.782403 3792 obj_retry.go:458] Detected object openshift-console/console-796968dcf4-zd56g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:58:19.782429 3792 obj_retry.go:458] Detected object openshift-console/console-796968dcf4-zd56g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:58:19.790162 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-796968dcf4-zd56g I0331 03:58:19.790190 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-796968dcf4-zd56g I0331 04:00:04.201951 3792 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.202369 3792 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.620347 3792 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.620908 3792 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:05.210100 3792 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.210123 3792 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-x57cm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.506894 3792 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.506949 3792 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-jb75l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:42.565128 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=99 I0331 04:00:44.656174 3792 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0331 04:00:45.514117 3792 namespace.go:98] [tsf] adding namespace I0331 04:00:45.515062 3792 namespace.go:102] [tsf] adding namespace took 886.962µs I0331 04:00:45.528603 3792 namespace.go:142] [tsf] updating namespace I0331 04:00:45.541166 3792 namespace.go:142] [tsf] updating namespace I0331 04:00:49.698559 3792 namespace.go:98] [tsf-tpa] adding namespace I0331 04:00:49.698578 3792 namespace.go:98] [rhtpa-operator] adding namespace I0331 04:00:49.698739 3792 namespace.go:98] [tsf-tas] adding namespace I0331 04:00:49.698813 3792 namespace.go:98] [konflux-ui] adding namespace I0331 04:00:49.699513 3792 namespace.go:102] [tsf-tpa] adding namespace took 923.124µs I0331 04:00:49.700051 3792 namespace.go:102] [rhtpa-operator] adding namespace took 1.455331ms I0331 04:00:49.700568 3792 namespace.go:102] [tsf-tas] adding namespace took 1.81501ms I0331 04:00:49.701002 3792 namespace.go:102] [konflux-ui] adding namespace took 2.179136ms I0331 04:00:49.702405 3792 namespace.go:98] [rhbk-operator] adding namespace I0331 04:00:49.702518 3792 namespace.go:98] [cert-manager-operator] adding namespace I0331 04:00:49.702585 3792 namespace.go:98] [konflux-operator] adding namespace I0331 04:00:49.702642 3792 namespace.go:98] [tsf-keycloak] adding namespace I0331 04:00:49.703102 3792 namespace.go:102] [rhbk-operator] adding namespace took 681.862µs I0331 04:00:49.703744 3792 namespace.go:102] [cert-manager-operator] adding namespace took 1.209865ms I0331 04:00:49.704299 3792 namespace.go:102] [konflux-operator] adding namespace took 1.699243ms I0331 04:00:49.704775 3792 namespace.go:102] [tsf-keycloak] adding namespace took 2.120321ms I0331 04:00:49.714386 3792 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:49.728054 3792 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:49.733793 3792 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:00:49.735215 3792 namespace.go:142] [rhbk-operator] updating namespace I0331 04:00:49.735295 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:00:49.740683 3792 namespace.go:142] [tsf-tas] updating namespace I0331 04:00:49.750584 3792 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:00:49.754167 3792 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:00:49.754251 3792 namespace.go:142] [konflux-operator] updating namespace I0331 04:00:49.754596 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:49.754626 3792 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:49.788604 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:00:49.797764 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:00:49.806356 3792 namespace.go:142] [rhbk-operator] updating namespace I0331 04:00:49.818121 3792 namespace.go:142] [rhbk-operator] updating namespace I0331 04:00:49.829890 3792 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:00:49.840983 3792 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:00:49.848634 3792 namespace.go:142] [konflux-operator] updating namespace I0331 04:00:49.857936 3792 namespace.go:142] [konflux-operator] updating namespace I0331 04:00:49.867409 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:49.879198 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:49.883923 3792 namespace.go:142] [tsf-tas] updating namespace I0331 04:00:49.894756 3792 namespace.go:142] [tsf-tas] updating namespace I0331 04:00:49.903837 3792 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:00:50.816740 3792 namespace.go:142] [tsf-tpa] updating namespace I0331 04:00:50.827968 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:00:59.870291 3792 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.870826 3792 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.870847 3792 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.870855 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:00:59.870867 3792 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:00:59.870871 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:00.894795 3792 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:00.894821 3792 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:00.894835 3792 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:00.894850 3792 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7pc6x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:01.615564 3792 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.616075 3792 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.616100 3792 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.616108 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:01.616119 3792 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:01.616125 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:02.601284 3792 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.601794 3792 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.601821 3792 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.601828 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:02.601839 3792 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:02.601843 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:02.643690 3792 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:02.643714 3792 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:02.643728 3792 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:02.643746 3792 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1p5mrn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.635816 3792 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.635838 3792 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.635852 3792 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:03.635862 3792 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8z7dnk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:04.672635 3792 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.673177 3792 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.673208 3792 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.673218 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:04.673239 3792 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.673246 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:04.704609 3792 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.705036 3792 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.705059 3792 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.705066 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:04.705076 3792 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.705081 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:04.734860 3792 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.735284 3792 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.735305 3792 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.735312 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:04.735322 3792 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:04.735326 3792 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:05.699882 3792 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.699904 3792 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.699918 3792 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.699932 3792 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5dg4zb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.730840 3792 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.730860 3792 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.730872 3792 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.730886 3792 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafq6mjz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760137 3792 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760176 3792 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760187 3792 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:05.760198 3792 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7hf422 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:09.076707 3792 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-nhm5w for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:09.076817 3792 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w I0331 04:01:09.101809 3792 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w] pod update took 25.010097ms I0331 04:01:09.101905 3792 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w] addLogicalPort annotation time took 25.112528ms I0331 04:01:09.103352 3792 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w] addLogicalPort took 26.666673ms, libovsdb time 1.045899ms I0331 04:01:09.433874 3792 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w 0d48816eea8457e2123978d5ac57a92560748921fd3262513cb3311ee08170c6 network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w 0d48816eea8457e2123978d5ac57a92560748921fd3262513cb3311ee08170c6 network default NAD default] I0331 04:01:09.442561 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-nhm5w, hostIfaceName: 0d48816eea8457e, network: default, NAD default, SandboxID: "0d48816eea8457e2123978d5ac57a92560748921fd3262513cb3311ee08170c6", PCI device ID: , UID: "317c257f-63a2-42f8-9fd6-a7f8dd19c7d0", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0331 04:01:09.486104 3792 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w 0d48816eea8457e2123978d5ac57a92560748921fd3262513cb3311ee08170c6 network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhm5w 0d48816eea8457e2123978d5ac57a92560748921fd3262513cb3311ee08170c6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:a6:95:78:98:bf\",\"name\":\"0d48816eea8457e\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe803420-59b6-48b3-b378-56d5caf56d8b\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:01:12.682307 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:15.572263 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:15.583802 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0331 04:01:15.797079 3792 namespace.go:142] [openshift-operators] updating namespace I0331 04:01:15.808550 3792 namespace.go:142] [openshift-operators] updating namespace W0331 04:01:16.175813 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:16.186776 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:16.196597 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:16.204701 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:01:19.838695 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:23.695734 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0331 04:01:24.788784 3792 namespace.go:142] [openshift-operators] updating namespace I0331 04:01:25.364008 3792 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-lvzzf for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:25.364132 3792 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf I0331 04:01:25.397773 3792 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf] pod update took 33.667926ms I0331 04:01:25.397795 3792 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf] addLogicalPort annotation time took 33.695501ms I0331 04:01:25.398984 3792 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf] addLogicalPort took 34.994965ms, libovsdb time 880.068µs I0331 04:01:25.738960 3792 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf 5ba9d997045373b06b7c31c95e960248f38613418dc7105f7dfe750ff8cceee7 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf 5ba9d997045373b06b7c31c95e960248f38613418dc7105f7dfe750ff8cceee7 network default NAD default] I0331 04:01:25.747427 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-lvzzf, hostIfaceName: 5ba9d997045373b, network: default, NAD default, SandboxID: "5ba9d997045373b06b7c31c95e960248f38613418dc7105f7dfe750ff8cceee7", PCI device ID: , UID: "e0580c05-4807-4efb-a876-1b921f475c96", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0331 04:01:25.998091 3792 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf 5ba9d997045373b06b7c31c95e960248f38613418dc7105f7dfe750ff8cceee7 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-lvzzf 5ba9d997045373b06b7c31c95e960248f38613418dc7105f7dfe750ff8cceee7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:cd:11:4e:ca:f7\",\"name\":\"5ba9d997045373b\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1e8e561c-24c8-4546-986e-f450ac8de1b4\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:01:28.347821 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:28.407157 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:28.467937 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:01:28.486688 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:01:31.073389 3792 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-5x6z7 for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:31.073571 3792 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7 I0331 04:01:31.116957 3792 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7] pod update took 43.4254ms I0331 04:01:31.116980 3792 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7] addLogicalPort annotation time took 43.452785ms I0331 04:01:31.118219 3792 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7] addLogicalPort took 44.850174ms, libovsdb time 907.287µs I0331 04:01:31.434059 3792 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7 9534d84bce8ea504f5be6fb5d588e2668b16d207e976ac76b1074ca88575dd1c network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7 9534d84bce8ea504f5be6fb5d588e2668b16d207e976ac76b1074ca88575dd1c network default NAD default] I0331 04:01:31.442358 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-5x6z7, hostIfaceName: 9534d84bce8ea50, network: default, NAD default, SandboxID: "9534d84bce8ea504f5be6fb5d588e2668b16d207e976ac76b1074ca88575dd1c", PCI device ID: , UID: "ac96fad2-0218-44d0-a63b-3fd27b3472a4", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0331 04:01:31.486579 3792 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7 9534d84bce8ea504f5be6fb5d588e2668b16d207e976ac76b1074ca88575dd1c network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-5x6z7 9534d84bce8ea504f5be6fb5d588e2668b16d207e976ac76b1074ca88575dd1c network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:44:35:2e:bc:f3\",\"name\":\"9534d84bce8ea50\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4710902b-afb9-4c79-984f-639ef0165690\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:01:34.103039 3792 namespace.go:98] [trusted-artifact-signer] adding namespace I0331 04:01:34.103857 3792 namespace.go:102] [trusted-artifact-signer] adding namespace took 794.819µs I0331 04:01:34.134746 3792 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:34.142688 3792 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:34.271045 3792 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-2kqrk for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:34.271138 3792 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk I0331 04:01:34.285130 3792 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk] pod update took 14.015963ms I0331 04:01:34.285170 3792 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk] addLogicalPort annotation time took 14.062565ms I0331 04:01:34.286400 3792 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk] addLogicalPort took 15.370118ms, libovsdb time 887.905µs I0331 04:01:34.631529 3792 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk 1fd7673ecb59e65f825553b9874bda0e9f4dc79299dbef58c68bb9245e845a12 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk 1fd7673ecb59e65f825553b9874bda0e9f4dc79299dbef58c68bb9245e845a12 network default NAD default] I0331 04:01:34.641136 3792 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-2kqrk, hostIfaceName: 1fd7673ecb59e65, network: default, NAD default, SandboxID: "1fd7673ecb59e65f825553b9874bda0e9f4dc79299dbef58c68bb9245e845a12", PCI device ID: , UID: "d75fa5aa-c602-4146-b47c-6dbda2d76e78", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0331 04:01:34.683453 3792 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk 1fd7673ecb59e65f825553b9874bda0e9f4dc79299dbef58c68bb9245e845a12 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-2kqrk 1fd7673ecb59e65f825553b9874bda0e9f4dc79299dbef58c68bb9245e845a12 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:d9:f6:e8:f7:05\",\"name\":\"1fd7673ecb59e65\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/477c10af-da2c-4cd7-a5a0-5d4656cd4616\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:01:34.791363 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0331 04:01:39.243966 3792 namespace.go:98] [cert-manager] adding namespace I0331 04:01:39.244888 3792 namespace.go:102] [cert-manager] adding namespace took 901.313µs I0331 04:01:39.277158 3792 namespace.go:142] [cert-manager] updating namespace I0331 04:01:39.291836 3792 namespace.go:142] [cert-manager] updating namespace I0331 04:01:40.660879 3792 namespace.go:98] [openshift-pipelines] adding namespace I0331 04:01:40.661725 3792 namespace.go:102] [openshift-pipelines] adding namespace took 824.643µs I0331 04:01:40.718798 3792 namespace.go:142] [openshift-pipelines] updating namespace I0331 04:01:40.722888 3792 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-k6tcl for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:40.722998 3792 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl I0331 04:01:40.779729 3792 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl] pod update took 56.763079ms I0331 04:01:40.779755 3792 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl] addLogicalPort annotation time took 56.794673ms I0331 04:01:40.781117 3792 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl] addLogicalPort took 58.247442ms, libovsdb time 1.011813ms I0331 04:01:41.087080 3792 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl b95ac04f968cb67ea5065e6212d41b405a9d59a0eebf1cd251cca361b01b9f2d network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl b95ac04f968cb67ea5065e6212d41b405a9d59a0eebf1cd251cca361b01b9f2d network default NAD default] I0331 04:01:41.096695 3792 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-k6tcl, hostIfaceName: b95ac04f968cb67, network: default, NAD default, SandboxID: "b95ac04f968cb67ea5065e6212d41b405a9d59a0eebf1cd251cca361b01b9f2d", PCI device ID: , UID: "0399160e-02c9-44bf-b75e-50ae0e03a034", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0331 04:01:41.147120 3792 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl b95ac04f968cb67ea5065e6212d41b405a9d59a0eebf1cd251cca361b01b9f2d network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-k6tcl b95ac04f968cb67ea5065e6212d41b405a9d59a0eebf1cd251cca361b01b9f2d network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:8b:fc:ed:04:e6\",\"name\":\"b95ac04f968cb67\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/62a48883-d561-4952-9702-c745d5f39420\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:01:41.587837 3792 namespace.go:142] [cert-manager] updating namespace I0331 04:01:41.794880 3792 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:42.203590 3792 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:42.402874 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:42.608872 3792 namespace.go:142] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] updating namespace I0331 04:01:43.063768 3792 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:43.276910 3792 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:43.477783 3792 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:43.687818 3792 namespace.go:142] [tsf] updating namespace I0331 04:01:43.875827 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:44.083885 3792 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:44.286672 3792 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:44.417465 3792 namespace.go:142] [cert-manager] updating namespace I0331 04:01:44.426308 3792 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:44.439338 3792 namespace.go:142] [default] updating namespace I0331 04:01:44.450018 3792 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:44.459715 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:44.474862 3792 namespace.go:142] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] updating namespace I0331 04:01:44.480803 3792 namespace.go:142] [openshift] updating namespace I0331 04:01:44.498532 3792 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:44.512690 3792 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:44.521196 3792 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:01:44.531930 3792 namespace.go:142] [tsf] updating namespace I0331 04:01:44.541547 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:44.553525 3792 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:44.564638 3792 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:44.660434 3792 namespace.go:142] [cert-manager] updating namespace I0331 04:01:44.767801 3792 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:44.859347 3792 namespace.go:142] [default] updating namespace I0331 04:01:44.961823 3792 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:45.058789 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:45.159418 3792 namespace.go:142] [open-cluster-management-2pcf4jph0npl5lb2s9sicapsrbqpq2gt] updating namespace I0331 04:01:45.332589 3792 namespace.go:142] [openshift] updating namespace I0331 04:01:45.451779 3792 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:45.546820 3792 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:45.668668 3792 namespace.go:142] [trusted-artifact-signer] updating namespace W0331 04:01:45.805964 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0331 04:01:45.824774 3792 namespace.go:142] [tsf] updating namespace I0331 04:01:45.929996 3792 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:46.052865 3792 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:46.152019 3792 namespace.go:142] [tsf-tpa] updating namespace W0331 04:01:46.968475 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:01:46.968495 3792 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0331 04:01:47.085938 3792 namespace.go:142] [openshift-pipelines] updating namespace W0331 04:01:52.873710 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:01:52.873739 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0331 04:01:54.541820 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-zd44s for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:54.541925 3792 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s I0331 04:01:54.554918 3792 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s] pod update took 13.020694ms I0331 04:01:54.554946 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s] addLogicalPort annotation time took 13.052864ms I0331 04:01:54.556275 3792 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s] addLogicalPort took 14.474957ms, libovsdb time 966.457µs I0331 04:01:54.663284 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-vhbnr for pod on switch ip-10-0-1-138.ec2.internal I0331 04:01:54.663384 3792 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr I0331 04:01:54.681845 3792 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr] pod update took 18.491058ms I0331 04:01:54.681875 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr] addLogicalPort annotation time took 18.526112ms I0331 04:01:54.683123 3792 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr] addLogicalPort took 19.852094ms, libovsdb time 826.807µs I0331 04:01:54.902108 3792 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s 6249079b5ba83eda995dfccb01497f6dcaf07bc4c4c5baea8331e878b7f60b7d network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s 6249079b5ba83eda995dfccb01497f6dcaf07bc4c4c5baea8331e878b7f60b7d network default NAD default] I0331 04:01:54.911008 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-zd44s, hostIfaceName: 6249079b5ba83ed, network: default, NAD default, SandboxID: "6249079b5ba83eda995dfccb01497f6dcaf07bc4c4c5baea8331e878b7f60b7d", PCI device ID: , UID: "734b9b53-81c1-4e2a-8277-2d846d37ba8c", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0331 04:01:54.957022 3792 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s 6249079b5ba83eda995dfccb01497f6dcaf07bc4c4c5baea8331e878b7f60b7d network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-zd44s 6249079b5ba83eda995dfccb01497f6dcaf07bc4c4c5baea8331e878b7f60b7d network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:5d:11:eb:e8:98\",\"name\":\"6249079b5ba83ed\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d971c216-ad28-4387-bd4d-f3603afdceac\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:01:55.023746 3792 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr 3dc2d3dd31c6dfca1a52738f4ba022a41e03407745dbb22bcd6185590dd1dd7b network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr 3dc2d3dd31c6dfca1a52738f4ba022a41e03407745dbb22bcd6185590dd1dd7b network default NAD default] I0331 04:01:55.032923 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-vhbnr, hostIfaceName: 3dc2d3dd31c6dfc, network: default, NAD default, SandboxID: "3dc2d3dd31c6dfca1a52738f4ba022a41e03407745dbb22bcd6185590dd1dd7b", PCI device ID: , UID: "da35e4ff-72f2-4fd5-abc4-1a497a7e5399", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0331 04:01:55.079690 3792 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr 3dc2d3dd31c6dfca1a52738f4ba022a41e03407745dbb22bcd6185590dd1dd7b network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-vhbnr 3dc2d3dd31c6dfca1a52738f4ba022a41e03407745dbb22bcd6185590dd1dd7b network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:e5:83:1d:98:dc\",\"name\":\"3dc2d3dd31c6dfc\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/43d1de22-ff55-4579-a7b7-ef11655ecbc8\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:01:57.909536 3792 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:57.909556 3792 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:57.909563 3792 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:57.909567 3792 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:01:59.080696 3792 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:01:59.080749 3792 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:01:59.080757 3792 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:01:59.107952 3792 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:01:59.107968 3792 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:01:59.107974 3792 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0331 04:02:02.666290 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0331 04:02:03.957856 3792 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:03.958402 3792 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:03.983717 3792 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:03.983738 3792 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:03.991130 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0331 04:02:03.991173 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0331 04:02:08.917188 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:08.917214 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:08.917220 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:08.917226 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0331 04:02:12.547502 3792 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-138.ec2.internal I0331 04:02:12.547529 3792 egressfirewall.go:628] Syncing node "ip-10-0-1-138.ec2.internal" for egress firewall I0331 04:02:12.547540 3792 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-138.ec2.internal: 46.22µs I0331 04:02:12.577969 3792 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-89wjr] creating logical port kube-system_global-pull-secret-syncer-89wjr for pod on switch ip-10-0-1-138.ec2.internal I0331 04:02:12.578065 3792 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-89wjr I0331 04:02:12.589828 3792 pod.go:62] [kube-system/global-pull-secret-syncer-89wjr] pod update took 11.784599ms I0331 04:02:12.589852 3792 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-89wjr] addLogicalPort annotation time took 11.813816ms I0331 04:02:12.591312 3792 pods.go:271] [kube-system/global-pull-secret-syncer-89wjr] addLogicalPort took 13.364653ms, libovsdb time 1.099453ms I0331 04:02:12.945357 3792 cni.go:403] [kube-system/global-pull-secret-syncer-89wjr 3b12b67edfdc7339f8a7af098b8b61cd9dc1fff9449d978074758d127453fdc0 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-89wjr 3b12b67edfdc7339f8a7af098b8b61cd9dc1fff9449d978074758d127453fdc0 network default NAD default] I0331 04:02:12.953988 3792 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-89wjr, hostIfaceName: 3b12b67edfdc733, network: default, NAD default, SandboxID: "3b12b67edfdc7339f8a7af098b8b61cd9dc1fff9449d978074758d127453fdc0", PCI device ID: , UID: "49cf291d-a7f3-4cd9-9588-f6d310d48465", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0331 04:02:12.999593 3792 cni.go:424] [kube-system/global-pull-secret-syncer-89wjr 3b12b67edfdc7339f8a7af098b8b61cd9dc1fff9449d978074758d127453fdc0 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-89wjr 3b12b67edfdc7339f8a7af098b8b61cd9dc1fff9449d978074758d127453fdc0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:4e:75:94:ff:be\",\"name\":\"3b12b67edfdc733\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e17dc6ab-5d15-4b20-9b0f-134303b22f75\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:02:13.888891 3792 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0331 04:02:13.912431 3792 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0331 04:02:20.336547 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-ztlt2 for pod on switch ip-10-0-1-138.ec2.internal I0331 04:02:20.336681 3792 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2 I0331 04:02:20.349161 3792 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2] pod update took 12.490239ms I0331 04:02:20.349185 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2] addLogicalPort annotation time took 12.538621ms I0331 04:02:20.350397 3792 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2] addLogicalPort took 13.868302ms, libovsdb time 814.144µs I0331 04:02:20.761480 3792 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2 f0d365d15de312d87f105e682483a1b8c0ec33f0c3bebf8d42cf7b6451b4a2af network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2 f0d365d15de312d87f105e682483a1b8c0ec33f0c3bebf8d42cf7b6451b4a2af network default NAD default] I0331 04:02:20.770220 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-ztlt2, hostIfaceName: f0d365d15de312d, network: default, NAD default, SandboxID: "f0d365d15de312d87f105e682483a1b8c0ec33f0c3bebf8d42cf7b6451b4a2af", PCI device ID: , UID: "dc0fa450-0597-4da7-a37c-f5fd7d9ec1c2", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0331 04:02:20.817228 3792 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2 f0d365d15de312d87f105e682483a1b8c0ec33f0c3bebf8d42cf7b6451b4a2af network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-ztlt2 f0d365d15de312d87f105e682483a1b8c0ec33f0c3bebf8d42cf7b6451b4a2af network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:ce:7c:06:e5:8d\",\"name\":\"f0d365d15de312d\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/83e004b7-b45e-4a89-8dcc-702df36043a5\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:02:22.596818 3792 namespace.go:142] [openshift-pipelines] updating namespace W0331 04:02:23.035236 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0331 04:02:29.678372 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 W0331 04:02:34.053521 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0331 04:02:36.695686 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:02:37.143968 3792 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:37.144602 3792 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:37.164194 3792 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:37.164216 3792 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:37.170086 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0331 04:02:37.170108 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0331 04:02:40.997708 3792 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0331 04:02:44.072413 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:02:44.074172 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:02:44.074190 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0331 04:02:44.077872 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-1-138.ec2.internal I0331 04:02:44.077956 3792 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0331 04:02:44.088958 3792 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 11.024283ms I0331 04:02:44.088979 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 11.050328ms I0331 04:02:44.090451 3792 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 12.595095ms, libovsdb time 1.111191ms I0331 04:02:54.348387 3792 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 97d541a0d8b898290e3ea79067d8019d58429a9397e73fea1b4a0479a1abf95a network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 97d541a0d8b898290e3ea79067d8019d58429a9397e73fea1b4a0479a1abf95a network default NAD default] I0331 04:02:54.357104 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 97d541a0d8b8982, network: default, NAD default, SandboxID: "97d541a0d8b898290e3ea79067d8019d58429a9397e73fea1b4a0479a1abf95a", PCI device ID: , UID: "1fe66ba2-7d24-4442-8448-b0560f848080", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0331 04:02:54.402361 3792 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 97d541a0d8b898290e3ea79067d8019d58429a9397e73fea1b4a0479a1abf95a network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 97d541a0d8b898290e3ea79067d8019d58429a9397e73fea1b4a0479a1abf95a network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:6d:79:52:23:c2\",\"name\":\"97d541a0d8b8982\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f8c4108f-5858-43a4-9200-ba6bec6e9e69\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:03:00.265115 3792 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0331 04:03:01.579374 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=625 I0331 04:03:01.698514 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-55cbb85cb8-mqthg for pod on switch ip-10-0-1-138.ec2.internal I0331 04:03:01.698602 3792 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg I0331 04:03:01.713451 3792 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg] pod update took 14.869114ms I0331 04:03:01.713474 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg] addLogicalPort annotation time took 14.896959ms I0331 04:03:01.714665 3792 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg] addLogicalPort took 16.169821ms, libovsdb time 841.944µs I0331 04:03:01.802891 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-nhfgt for pod on switch ip-10-0-1-138.ec2.internal I0331 04:03:01.803012 3792 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt I0331 04:03:01.816410 3792 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt] pod update took 13.42185ms I0331 04:03:01.816431 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt] addLogicalPort annotation time took 13.448369ms I0331 04:03:01.817713 3792 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt] addLogicalPort took 14.83632ms, libovsdb time 875.033µs I0331 04:03:02.057311 3792 cni.go:403] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg e181f1f78fa52dd286906bd95329eb4248444b4389f6cf6477dfec5ae3be664c network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg e181f1f78fa52dd286906bd95329eb4248444b4389f6cf6477dfec5ae3be664c network default NAD default] I0331 04:03:02.065902 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-55cbb85cb8-mqthg, hostIfaceName: e181f1f78fa52dd, network: default, NAD default, SandboxID: "e181f1f78fa52dd286906bd95329eb4248444b4389f6cf6477dfec5ae3be664c", PCI device ID: , UID: "886b0f3c-19f7-4172-ac4d-3d15f3d98df2", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0331 04:03:02.112193 3792 cni.go:424] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg e181f1f78fa52dd286906bd95329eb4248444b4389f6cf6477dfec5ae3be664c network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-mqthg e181f1f78fa52dd286906bd95329eb4248444b4389f6cf6477dfec5ae3be664c network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:ff:11:c8:aa:cd\",\"name\":\"e181f1f78fa52dd\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50fe1ca5-465a-4916-b346-4104c59f026e\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:03:02.162383 3792 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt 5c3a238804dfd3db0a9a958c84e0da1b4a27a000423add20e222606b05e656b3 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt 5c3a238804dfd3db0a9a958c84e0da1b4a27a000423add20e222606b05e656b3 network default NAD default] I0331 04:03:02.171188 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-nhfgt, hostIfaceName: 5c3a238804dfd3d, network: default, NAD default, SandboxID: "5c3a238804dfd3db0a9a958c84e0da1b4a27a000423add20e222606b05e656b3", PCI device ID: , UID: "507d4543-f3cf-4d35-82b1-5ce6075b2f61", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0331 04:03:02.216682 3792 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt 5c3a238804dfd3db0a9a958c84e0da1b4a27a000423add20e222606b05e656b3 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-nhfgt 5c3a238804dfd3db0a9a958c84e0da1b4a27a000423add20e222606b05e656b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:9e:4f:fc:55:c4\",\"name\":\"5c3a238804dfd3d\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9bc09596-e383-4c0e-9f67-952539823016\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:03:08.213843 3792 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:08.213867 3792 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:03:08.213874 3792 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0331 04:03:12.424085 3792 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:12.424658 3792 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:12.583827 3792 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:12.583851 3792 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:12.588753 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0331 04:03:12.588779 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0331 04:03:12.672509 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0331 04:03:12.697395 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0331 04:03:14.417566 3792 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-ghcnp-74sdd] creating logical port tsf-tas_segment-backup-installation-ghcnp-74sdd for pod on switch ip-10-0-1-138.ec2.internal I0331 04:03:14.417669 3792 kube.go:256] Updating pod tsf-tas/segment-backup-installation-ghcnp-74sdd I0331 04:03:14.428734 3792 pod.go:62] [tsf-tas/segment-backup-installation-ghcnp-74sdd] pod update took 11.08924ms I0331 04:03:14.428758 3792 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-ghcnp-74sdd] addLogicalPort annotation time took 11.122482ms I0331 04:03:14.430242 3792 pods.go:271] [tsf-tas/segment-backup-installation-ghcnp-74sdd] addLogicalPort took 12.691478ms, libovsdb time 981.649µs I0331 04:03:14.775670 3792 cni.go:403] [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default] I0331 04:03:14.785238 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-ghcnp-74sdd, hostIfaceName: 5cd125f0932cab2, network: default, NAD default, SandboxID: "5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392", PCI device ID: , UID: "2984d0ea-de39-4333-a2b7-68fb15b31d34", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0331 04:03:14.832242 3792 cni.go:424] [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:18:48:fd:4f:34\",\"name\":\"5cd125f0932cab2\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3a7899b-b843-4d73-abd6-77a543f27b00\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:03:15.660380 3792 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-78cdf58997-s6vbr] creating logical port tsf-tas_trillian-logsigner-78cdf58997-s6vbr for pod on switch ip-10-0-1-138.ec2.internal I0331 04:03:15.660585 3792 kube.go:256] Updating pod tsf-tas/trillian-logsigner-78cdf58997-s6vbr I0331 04:03:15.676138 3792 pod.go:62] [tsf-tas/trillian-logsigner-78cdf58997-s6vbr] pod update took 15.59137ms I0331 04:03:15.676223 3792 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logsigner-78cdf58997-s6vbr] addLogicalPort annotation time took 15.682701ms I0331 04:03:15.679263 3792 pods.go:271] [tsf-tas/trillian-logsigner-78cdf58997-s6vbr] addLogicalPort took 18.902958ms, libovsdb time 2.54324ms I0331 04:03:16.563400 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0331 04:03:16.636535 3792 cni.go:403] [tsf-tas/trillian-logsigner-78cdf58997-s6vbr ca1acc91ea0ec55e5d0c95126490c415c4235e86c3b5b2e35321ed86bcf3e8e0 network default NAD default] ADD starting CNI request [tsf-tas/trillian-logsigner-78cdf58997-s6vbr ca1acc91ea0ec55e5d0c95126490c415c4235e86c3b5b2e35321ed86bcf3e8e0 network default NAD default] I0331 04:03:16.646191 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-78cdf58997-s6vbr, hostIfaceName: ca1acc91ea0ec55, network: default, NAD default, SandboxID: "ca1acc91ea0ec55e5d0c95126490c415c4235e86c3b5b2e35321ed86bcf3e8e0", PCI device ID: , UID: "6d0f820e-1f3c-4876-8eca-dbc1ab65fbe0", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0331 04:03:16.700356 3792 cni.go:424] [tsf-tas/trillian-logsigner-78cdf58997-s6vbr ca1acc91ea0ec55e5d0c95126490c415c4235e86c3b5b2e35321ed86bcf3e8e0 network default NAD default] ADD finished CNI request [tsf-tas/trillian-logsigner-78cdf58997-s6vbr ca1acc91ea0ec55e5d0c95126490c415c4235e86c3b5b2e35321ed86bcf3e8e0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:5a:2a:bb:da:aa\",\"name\":\"ca1acc91ea0ec55\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea879f46-1fc5-4d94-b936-0898aab5747c\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:03:19.439915 3792 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:03:19.439939 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:03:23.458348 3792 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0331 04:03:25.468706 3792 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-8tlcq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:25.469330 3792 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-8tlcq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 04:03:26.471322 3792 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0331 04:03:26.493611 3792 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-8tlcq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:26.493635 3792 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-8tlcq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:26.848952 3792 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:26.849511 3792 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:27.339399 3792 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:27.339423 3792 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:27.343700 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0331 04:03:27.343725 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0331 04:03:27.453284 3792 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0331 04:03:29.366336 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:29.366362 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:29.369066 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0331 04:03:30.397566 3792 cni.go:403] [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default] I0331 04:03:30.483253 3792 cni.go:424] [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-ghcnp-74sdd 5cd125f0932cab2c86c63cec70d20509601c458a1c41c72d1cda9e1543bb4392 network default NAD default], result "{}", err I0331 04:03:30.531813 3792 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:30.531853 3792 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-ghcnp-74sdd I0331 04:03:30.533562 3792 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-ghcnp-74sdd, ips: 10.130.0.33 I0331 04:03:30.533592 3792 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:31.552364 3792 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:31.552388 3792 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ghcnp-74sdd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:52.189727 3792 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-hfr6b for pod on switch ip-10-0-1-138.ec2.internal I0331 04:03:52.189884 3792 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b I0331 04:03:52.205058 3792 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b] pod update took 15.199917ms I0331 04:03:52.205086 3792 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b] addLogicalPort annotation time took 15.23228ms I0331 04:03:52.206494 3792 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b] addLogicalPort took 16.789619ms, libovsdb time 874.948µs I0331 04:03:52.833002 3792 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b 9fa77b8687962d0d195f3a3a8139cec0534d339f4f95ba49ad12aad3328840da network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b 9fa77b8687962d0d195f3a3a8139cec0534d339f4f95ba49ad12aad3328840da network default NAD default] I0331 04:03:52.841690 3792 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-hfr6b, hostIfaceName: 9fa77b8687962d0, network: default, NAD default, SandboxID: "9fa77b8687962d0d195f3a3a8139cec0534d339f4f95ba49ad12aad3328840da", PCI device ID: , UID: "1432a688-9588-48da-b4e5-e9dc2e977f73", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0331 04:03:52.888116 3792 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b 9fa77b8687962d0d195f3a3a8139cec0534d339f4f95ba49ad12aad3328840da network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hfr6b 9fa77b8687962d0d195f3a3a8139cec0534d339f4f95ba49ad12aad3328840da network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:db:00:f2:7b:27\",\"name\":\"9fa77b8687962d0\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6de235e0-10b7-487a-9c59-dcfb10416f0a\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:03:55.561712 3792 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:03:55.561734 3792 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:03:56.479684 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner W0331 04:03:56.528763 3792 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0331 04:04:02.700256 3792 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:02.700845 3792 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:02.720408 3792 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:02.720428 3792 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:02.726226 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0331 04:04:02.726254 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0331 04:04:03.364334 3792 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-xt9bv] creating logical port tsf_patch-tekton-config-xt9bv for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:03.364427 3792 kube.go:256] Updating pod tsf/patch-tekton-config-xt9bv I0331 04:04:03.377519 3792 pod.go:62] [tsf/patch-tekton-config-xt9bv] pod update took 13.116906ms I0331 04:04:03.377546 3792 base_network_controller_pods.go:937] [default/tsf/patch-tekton-config-xt9bv] addLogicalPort annotation time took 13.147232ms I0331 04:04:03.379053 3792 pods.go:271] [tsf/patch-tekton-config-xt9bv] addLogicalPort took 14.738717ms, libovsdb time 927.274µs I0331 04:04:03.751194 3792 cni.go:403] [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default] ADD starting CNI request [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default] I0331 04:04:03.760350 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-xt9bv, hostIfaceName: 36c832468642907, network: default, NAD default, SandboxID: "36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6", PCI device ID: , UID: "176183a5-8949-4b4e-99a7-a18b5d55cc47", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0331 04:04:03.806360 3792 cni.go:424] [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default] ADD finished CNI request [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:2d:97:8d:03:6e\",\"name\":\"36c832468642907\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6dbb0e2-d007-4103-8a39-b84aa1d09ff6\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:04:07.530989 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0331 04:04:07.683203 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0331 04:04:08.859953 3792 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:08.860562 3792 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:09.729891 3792 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:09.729921 3792 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jsjdx-56tsd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:10.563650 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=79 I0331 04:04:10.863227 3792 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:10.863848 3792 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:11.743744 3792 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:11.743771 3792 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-lx85r-jw9r2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:14.629426 3792 cni.go:403] [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default] DEL starting CNI request [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default] I0331 04:04:14.712257 3792 cni.go:424] [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default] DEL finished CNI request [tsf/patch-tekton-config-xt9bv 36c832468642907a6e386a1cd815c168bbff558d3583f3c2235fa13ffb09ebb6 network default NAD default], result "{}", err I0331 04:04:14.764226 3792 obj_retry.go:464] Detected object tsf/patch-tekton-config-xt9bv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:14.764265 3792 pods.go:176] Deleting pod: tsf/patch-tekton-config-xt9bv I0331 04:04:14.765668 3792 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-xt9bv, ips: 10.130.0.36 I0331 04:04:14.765698 3792 obj_retry.go:464] Detected object tsf/patch-tekton-config-xt9bv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:15.643138 3792 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:15.643187 3792 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:15.728723 3792 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-56768877f9-tvjgv] creating logical port tsf-tas_ctlog-56768877f9-tvjgv for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:15.728815 3792 kube.go:256] Updating pod tsf-tas/ctlog-56768877f9-tvjgv I0331 04:04:15.747219 3792 pod.go:62] [tsf-tas/ctlog-56768877f9-tvjgv] pod update took 18.429037ms I0331 04:04:15.747241 3792 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-56768877f9-tvjgv] addLogicalPort annotation time took 18.456211ms I0331 04:04:15.748722 3792 pods.go:271] [tsf-tas/ctlog-56768877f9-tvjgv] addLogicalPort took 20.019074ms, libovsdb time 930.922µs I0331 04:04:15.751514 3792 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-t5hq2] creating logical port tsf_tsf-tekton-configuration-t5hq2 for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:15.751604 3792 kube.go:256] Updating pod tsf/tsf-tekton-configuration-t5hq2 I0331 04:04:15.766611 3792 pod.go:62] [tsf/tsf-tekton-configuration-t5hq2] pod update took 15.028394ms I0331 04:04:15.766643 3792 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-t5hq2] addLogicalPort annotation time took 15.063789ms I0331 04:04:15.768722 3792 pods.go:271] [tsf/tsf-tekton-configuration-t5hq2] addLogicalPort took 17.222032ms, libovsdb time 1.482031ms I0331 04:04:16.116588 3792 cni.go:403] [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default] I0331 04:04:16.124971 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-t5hq2, hostIfaceName: b862552dde8a447, network: default, NAD default, SandboxID: "b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a", PCI device ID: , UID: "a295ec45-d89f-474c-8f9a-dbd1db66bf0c", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0331 04:04:16.172817 3792 cni.go:424] [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f6:e5:11:ab:45\",\"name\":\"b862552dde8a447\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ee78dc1-90c9-4186-8fad-f45290c9f909\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:04:16.692344 3792 cni.go:403] [tsf-tas/ctlog-56768877f9-tvjgv f97052541d2796dec0e20fa1f1447ab9f03abcb85905748af04514734bb81b81 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-56768877f9-tvjgv f97052541d2796dec0e20fa1f1447ab9f03abcb85905748af04514734bb81b81 network default NAD default] I0331 04:04:16.700754 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-56768877f9-tvjgv, hostIfaceName: f97052541d2796d, network: default, NAD default, SandboxID: "f97052541d2796dec0e20fa1f1447ab9f03abcb85905748af04514734bb81b81", PCI device ID: , UID: "909f5f16-6286-4840-ac76-5ac12c63168d", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0331 04:04:16.748123 3792 cni.go:424] [tsf-tas/ctlog-56768877f9-tvjgv f97052541d2796dec0e20fa1f1447ab9f03abcb85905748af04514734bb81b81 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-56768877f9-tvjgv f97052541d2796dec0e20fa1f1447ab9f03abcb85905748af04514734bb81b81 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:a2:0e:51:0a:a0\",\"name\":\"f97052541d2796d\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/12b2b8f6-b45d-46d8-9e59-c0a263658bbc\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:04:16.764954 3792 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0331 04:04:18.778990 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0331 04:04:19.672742 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0331 04:04:21.571457 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=103 I0331 04:04:24.692676 3792 cni.go:403] [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default] I0331 04:04:24.776939 3792 cni.go:424] [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-t5hq2 b862552dde8a447227b3b968bafe947d45c801f2dc7c1d077de09c90b9ad890a network default NAD default], result "{}", err I0331 04:04:24.824519 3792 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:24.824554 3792 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-t5hq2 I0331 04:04:24.825799 3792 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-t5hq2, ips: 10.130.0.38 I0331 04:04:24.825826 3792 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:25.843860 3792 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.843889 3792 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-t5hq2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.877118 3792 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.877160 3792 obj_retry.go:458] Detected object tsf/patch-tekton-config-xt9bv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:25.881799 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-xt9bv I0331 04:04:25.881831 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-xt9bv I0331 04:04:26.000899 3792 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:26.000992 3792 kube.go:256] Updating pod tsf/test-tsf-pipelines I0331 04:04:26.012214 3792 pod.go:62] [tsf/test-tsf-pipelines] pod update took 11.245627ms I0331 04:04:26.012240 3792 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 11.274548ms I0331 04:04:26.013761 3792 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 12.876828ms, libovsdb time 1.002122ms I0331 04:04:26.359441 3792 cni.go:403] [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default] I0331 04:04:26.367743 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: aa3461ed2dc7027, network: default, NAD default, SandboxID: "aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f", PCI device ID: , UID: "49aad971-e4b7-409c-9509-9bc79a7aa9fd", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0331 04:04:26.619688 3792 cni.go:424] [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:a3:90:46:98:73\",\"name\":\"aa3461ed2dc7027\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ff13aa51-e52d-42c0-bb82-e1267a9e47df\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:04:26.827604 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0331 04:04:30.673123 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0331 04:04:36.767510 3792 cni.go:403] [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default] I0331 04:04:36.835728 3792 cni.go:424] [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines aa3461ed2dc7027ac1fdd2458aadc49d906e027f266b439d927e39853167599f network default NAD default], result "{}", err I0331 04:04:36.884565 3792 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:36.884599 3792 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0331 04:04:36.885822 3792 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.130.0.39 I0331 04:04:36.885850 3792 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:36.901644 3792 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:36.901668 3792 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:36.909016 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0331 04:04:36.909039 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0331 04:04:40.672298 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0331 04:04:40.923913 3792 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:40.924461 3792 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:40.943163 3792 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:40.943182 3792 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:40.948976 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0331 04:04:40.949000 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0331 04:04:42.197553 3792 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-p8489] creating logical port tsf-tpa_create-db-p8489 for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:42.197636 3792 kube.go:256] Updating pod tsf-tpa/create-db-p8489 I0331 04:04:42.209989 3792 pod.go:62] [tsf-tpa/create-db-p8489] pod update took 12.37465ms I0331 04:04:42.210011 3792 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-p8489] addLogicalPort annotation time took 12.401985ms I0331 04:04:42.211351 3792 pods.go:271] [tsf-tpa/create-db-p8489] addLogicalPort took 13.814237ms, libovsdb time 911.032µs I0331 04:04:42.563506 3792 cni.go:403] [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default] ADD starting CNI request [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default] I0331 04:04:42.581217 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-p8489, hostIfaceName: 366a7262ed28e9e, network: default, NAD default, SandboxID: "366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51", PCI device ID: , UID: "65c87aea-7a59-4871-8292-21fe1f790f8b", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0331 04:04:42.631903 3792 cni.go:424] [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default] ADD finished CNI request [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:9c:7d:9a:2c:95\",\"name\":\"366a7262ed28e9e\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9498c60-f522-4ee0-b91c-f39f7becd893\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:04:48.842656 3792 cni.go:403] [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default] DEL starting CNI request [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default] I0331 04:04:48.936326 3792 cni.go:424] [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default] DEL finished CNI request [tsf-tpa/create-db-p8489 366a7262ed28e9ea337a831b724f5f1d6d06baff17bd3c6d53207d4750b5df51 network default NAD default], result "{}", err I0331 04:04:48.985429 3792 obj_retry.go:464] Detected object tsf-tpa/create-db-p8489 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:48.985467 3792 pods.go:176] Deleting pod: tsf-tpa/create-db-p8489 I0331 04:04:48.986815 3792 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-p8489, ips: 10.130.0.40 I0331 04:04:48.986841 3792 obj_retry.go:464] Detected object tsf-tpa/create-db-p8489 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:49.856546 3792 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:49.856572 3792 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:49.947681 3792 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-q4jqv] creating logical port tsf-tpa_migrate-db-q4jqv for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:49.947775 3792 kube.go:256] Updating pod tsf-tpa/migrate-db-q4jqv I0331 04:04:49.959132 3792 pod.go:62] [tsf-tpa/migrate-db-q4jqv] pod update took 11.378207ms I0331 04:04:49.959166 3792 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-q4jqv] addLogicalPort annotation time took 11.417321ms I0331 04:04:49.960588 3792 pods.go:271] [tsf-tpa/migrate-db-q4jqv] addLogicalPort took 12.921701ms, libovsdb time 1.003182ms I0331 04:04:50.307990 3792 cni.go:403] [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default] I0331 04:04:50.316518 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-q4jqv, hostIfaceName: f72759a7b4e6b13, network: default, NAD default, SandboxID: "f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319", PCI device ID: , UID: "887d9d37-f522-4767-82dd-60583d2af387", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0331 04:04:50.361443 3792 cni.go:424] [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:82:66:89:5d:45\",\"name\":\"f72759a7b4e6b13\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/830e6bbf-a2d8-403a-8d30-3a756002b270\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:04:52.866446 3792 cni.go:403] [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default] I0331 04:04:52.956858 3792 cni.go:424] [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-q4jqv f72759a7b4e6b131fb101b5cb774555ab43aa7cd3cec97496029d700b9df9319 network default NAD default], result "{}", err I0331 04:04:53.005695 3792 obj_retry.go:464] Detected object tsf-tpa/migrate-db-q4jqv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:53.005729 3792 pods.go:176] Deleting pod: tsf-tpa/migrate-db-q4jqv I0331 04:04:53.006989 3792 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-q4jqv, ips: 10.130.0.41 I0331 04:04:53.007016 3792 obj_retry.go:464] Detected object tsf-tpa/migrate-db-q4jqv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:54.027710 3792 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:54.027741 3792 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:57.075455 3792 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:57.076032 3792 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:57.163693 3792 base_network_controller_pods.go:487] [default/tsf-tpa/server-787f776df4-k6cdb] creating logical port tsf-tpa_server-787f776df4-k6cdb for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:57.163794 3792 kube.go:256] Updating pod tsf-tpa/server-787f776df4-k6cdb I0331 04:04:57.176413 3792 pod.go:62] [tsf-tpa/server-787f776df4-k6cdb] pod update took 12.646982ms I0331 04:04:57.176434 3792 base_network_controller_pods.go:937] [default/tsf-tpa/server-787f776df4-k6cdb] addLogicalPort annotation time took 12.673875ms I0331 04:04:57.177740 3792 pods.go:271] [tsf-tpa/server-787f776df4-k6cdb] addLogicalPort took 14.065415ms, libovsdb time 864.182µs I0331 04:04:57.571082 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=99 I0331 04:04:57.892722 3792 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:57.892749 3792 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-gk9fd-9nz9s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:59.009241 3792 obj_retry.go:464] Detected object tsf-tpa/create-importers-n6cv9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:59.009783 3792 obj_retry.go:464] Detected object tsf-tpa/create-importers-n6cv9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:59.021930 3792 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-kdjpm] creating logical port tsf-tpa_importer-74cd48f5c7-kdjpm for pod on switch ip-10-0-1-138.ec2.internal I0331 04:04:59.022026 3792 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-kdjpm I0331 04:04:59.035628 3792 pod.go:62] [tsf-tpa/importer-74cd48f5c7-kdjpm] pod update took 13.629438ms I0331 04:04:59.035656 3792 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-kdjpm] addLogicalPort annotation time took 13.65913ms I0331 04:04:59.036945 3792 pods.go:271] [tsf-tpa/importer-74cd48f5c7-kdjpm] addLogicalPort took 15.03338ms, libovsdb time 861.754µs I0331 04:05:00.033481 3792 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:00.033505 3792 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:00.655498 3792 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 W0331 04:05:05.930414 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0331 04:05:08.007613 3792 cni.go:403] [tsf-tpa/server-787f776df4-k6cdb fd40bbf24aa436164faecf6fa6addb549358a9e337d4f3f5b39f68f2a45e60eb network default NAD default] ADD starting CNI request [tsf-tpa/server-787f776df4-k6cdb fd40bbf24aa436164faecf6fa6addb549358a9e337d4f3f5b39f68f2a45e60eb network default NAD default] I0331 04:05:08.016291 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-787f776df4-k6cdb, hostIfaceName: fd40bbf24aa4361, network: default, NAD default, SandboxID: "fd40bbf24aa436164faecf6fa6addb549358a9e337d4f3f5b39f68f2a45e60eb", PCI device ID: , UID: "e742c3bb-68e7-4086-88df-8c2f8faea87d", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0331 04:05:08.039571 3792 cni.go:403] [tsf-tpa/importer-74cd48f5c7-kdjpm 3bf8d1e9890f45f3fd3d4740c64a9cc7fb879d1c5dcd5c47173ba40b9e2eeae4 network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-kdjpm 3bf8d1e9890f45f3fd3d4740c64a9cc7fb879d1c5dcd5c47173ba40b9e2eeae4 network default NAD default] I0331 04:05:08.049198 3792 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-kdjpm, hostIfaceName: 3bf8d1e9890f45f, network: default, NAD default, SandboxID: "3bf8d1e9890f45f3fd3d4740c64a9cc7fb879d1c5dcd5c47173ba40b9e2eeae4", PCI device ID: , UID: "da5560c0-b6fc-4f4d-9d0d-7552fd621fb6", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0331 04:05:08.068649 3792 cni.go:424] [tsf-tpa/server-787f776df4-k6cdb fd40bbf24aa436164faecf6fa6addb549358a9e337d4f3f5b39f68f2a45e60eb network default NAD default] ADD finished CNI request [tsf-tpa/server-787f776df4-k6cdb fd40bbf24aa436164faecf6fa6addb549358a9e337d4f3f5b39f68f2a45e60eb network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:62:7d:d8:ac:1d\",\"name\":\"fd40bbf24aa4361\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67c1b278-cce7-48a7-8085-ba06d5e29216\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:05:08.099058 3792 cni.go:424] [tsf-tpa/importer-74cd48f5c7-kdjpm 3bf8d1e9890f45f3fd3d4740c64a9cc7fb879d1c5dcd5c47173ba40b9e2eeae4 network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-kdjpm 3bf8d1e9890f45f3fd3d4740c64a9cc7fb879d1c5dcd5c47173ba40b9e2eeae4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:8e:5b:ed:95:14\",\"name\":\"3bf8d1e9890f45f\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d221360f-fc62-4ddd-b307-f455854683d7\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0331 04:05:08.984907 3792 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0331 04:05:19.989079 3792 util.go:959] Failed to build node endpoints for node ip-10-0-1-138.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0331 04:05:30.689442 3792 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0331 04:05:35.215048 3792 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:35.215661 3792 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:35.253270 3792 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:35.253294 3792 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:35.259687 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0331 04:05:35.259711 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0331 04:05:35.931426 3792 namespace.go:98] [build-service] adding namespace I0331 04:05:35.931551 3792 namespace.go:98] [integration-service] adding namespace I0331 04:05:35.931576 3792 namespace.go:98] [image-controller] adding namespace I0331 04:05:35.932404 3792 namespace.go:102] [build-service] adding namespace took 951.968µs I0331 04:05:35.933082 3792 namespace.go:102] [integration-service] adding namespace took 1.456745ms I0331 04:05:35.933672 3792 namespace.go:102] [image-controller] adding namespace took 2.036179ms I0331 04:05:35.962986 3792 namespace.go:142] [build-service] updating namespace I0331 04:05:35.983691 3792 namespace.go:142] [build-service] updating namespace I0331 04:05:35.999675 3792 namespace.go:142] [integration-service] updating namespace I0331 04:05:36.022562 3792 namespace.go:142] [integration-service] updating namespace I0331 04:05:36.026296 3792 namespace.go:142] [image-controller] updating namespace I0331 04:05:36.045195 3792 namespace.go:142] [image-controller] updating namespace I0331 04:05:36.157119 3792 namespace.go:142] [build-service] updating namespace I0331 04:05:36.179746 3792 namespace.go:142] [integration-service] updating namespace I0331 04:05:36.254881 3792 namespace.go:98] [release-service] adding namespace I0331 04:05:36.255009 3792 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-rx66m] creating logical port konflux-ui_tsf-konflux-public-key-rx66m for pod on switch ip-10-0-1-138.ec2.internal I0331 04:05:36.255134 3792 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-rx66m I0331 04:05:36.255784 3792 namespace.go:102] [release-service] adding namespace took 871.657µs I0331 04:05:36.264807 3792 namespace.go:142] [konflux-ui] updating namespace I0331 04:05:36.284745 3792 namespace.go:142] [release-service] updating namespace I0331 04:05:36.296636 3792 pod.go:62] [konflux-ui/tsf-konflux-public-key-rx66m] pod update took 41.529076ms I0331 04:05:36.296658 3792 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-rx66m] addLogicalPort annotation time took 41.556394ms I0331 04:05:36.297929 3792 pods.go:271] [konflux-ui/tsf-konflux-public-key-rx66m] addLogicalPort took 42.937702ms, libovsdb time 862.401µs I0331 04:05:36.321195 3792 namespace.go:142] [release-service] updating namespace I0331 04:05:36.376750 3792 namespace.go:98] [konflux-info] adding namespace I0331 04:05:36.377603 3792 namespace.go:102] [konflux-info] adding namespace took 821.904µs I0331 04:05:36.448889 3792 namespace.go:142] [konflux-info] updating namespace I0331 04:05:36.471729 3792 namespace.go:98] [namespace-lister] adding namespace I0331 04:05:36.472787 3792 namespace.go:102] [namespace-lister] adding namespace took 1.035227ms I0331 04:05:36.473089 3792 namespace.go:98] [enterprise-contract-service] adding namespace I0331 04:05:36.473741 3792 namespace.go:102] [enterprise-contract-service] adding namespace took 636.517µs I0331 04:05:36.491280 3792 namespace.go:142] [konflux-info] updating namespace I0331 04:05:36.501908 3792 namespace.go:142] [image-controller] updating namespace I0331 04:05:36.538066 3792 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:36.608053 3792 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:36.609605 3792 cni.go:403] [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default] I0331 04:05:36.618288 3792 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-rx66m, hostIfaceName: 9c2d0cca157b4b0, network: default, NAD default, SandboxID: "9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c", PCI device ID: , UID: "15b4b237-71c5-49cf-86a8-86a5de6ecea8", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0331 04:05:36.618759 3792 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:36.639841 3792 namespace.go:98] [default-tenant] adding namespace I0331 04:05:36.640797 3792 namespace.go:102] [default-tenant] adding namespace took 928.415µs I0331 04:05:36.671305 3792 cni.go:424] [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:8d:75:59:2e:72\",\"name\":\"9c2d0cca157b4b0\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf098a84-dc97-469e-800f-ac3ffa5cd127\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0331 04:05:36.689739 3792 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:36.702756 3792 namespace.go:98] [segment-bridge] adding namespace I0331 04:05:36.703571 3792 namespace.go:102] [segment-bridge] adding namespace took 793.286µs I0331 04:05:36.728751 3792 namespace.go:142] [default-tenant] updating namespace I0331 04:05:36.781007 3792 namespace.go:142] [default-tenant] updating namespace I0331 04:05:36.837243 3792 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:36.882020 3792 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:37.049777 3792 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0331 04:05:37.050488 3792 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0331 04:05:37.050509 3792 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0331 04:05:37.053332 3792 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0331 04:05:37.192855 3792 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0331 04:05:37.192890 3792 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0331 04:05:37.194017 3792 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0331 04:05:37.688715 3792 namespace.go:142] [build-service] updating namespace I0331 04:05:38.317775 3792 namespace.go:142] [image-controller] updating namespace I0331 04:05:38.766210 3792 namespace.go:142] [integration-service] updating namespace I0331 04:05:39.055968 3792 namespace.go:142] [konflux-info] updating namespace I0331 04:05:39.132664 3792 cni.go:403] [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default] I0331 04:05:39.209943 3792 cni.go:424] [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-rx66m 9c2d0cca157b4b003056923b71ed1550141bef64a078493028cd81559508ca5c network default NAD default], result "{}", err I0331 04:05:39.253750 3792 namespace.go:142] [release-service] updating namespace I0331 04:05:39.279109 3792 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:39.279163 3792 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-rx66m I0331 04:05:39.280494 3792 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-rx66m, ips: 10.130.0.44 I0331 04:05:39.280523 3792 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:39.385887 3792 namespace.go:142] [build-service] updating namespace I0331 04:05:39.398431 3792 namespace.go:142] [image-controller] updating namespace I0331 04:05:39.414414 3792 namespace.go:142] [integration-service] updating namespace I0331 04:05:39.451979 3792 namespace.go:142] [konflux-info] updating namespace I0331 04:05:39.474935 3792 namespace.go:142] [release-service] updating namespace I0331 04:05:39.545406 3792 namespace.go:142] [build-service] updating namespace I0331 04:05:39.616121 3792 namespace.go:142] [image-controller] updating namespace I0331 04:05:39.683842 3792 namespace.go:142] [integration-service] updating namespace I0331 04:05:39.803485 3792 namespace.go:142] [konflux-info] updating namespace I0331 04:05:40.008661 3792 namespace.go:142] [release-service] updating namespace W0331 04:05:40.244857 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0331 04:05:40.305250 3792 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:40.305276 3792 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-rx66m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:41.161738 3792 namespace.go:142] [default-tenant] updating namespace I0331 04:05:41.362616 3792 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:41.562049 3792 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:41.767435 3792 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:41.884243 3792 namespace.go:142] [default-tenant] updating namespace I0331 04:05:41.896392 3792 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:41.909617 3792 namespace.go:142] [namespace-lister] updating namespace I0331 04:05:41.921547 3792 namespace.go:142] [segment-bridge] updating namespace I0331 04:05:41.993225 3792 namespace.go:142] [default-tenant] updating namespace I0331 04:05:42.095462 3792 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:05:42.191623 3792 namespace.go:142] [namespace-lister] updating namespace W0331 04:05:42.244247 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0331 04:05:42.293079 3792 namespace.go:142] [segment-bridge] updating namespace W0331 04:05:43.142233 3792 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0331 04:05:53.324840 3792 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0331 04:05:53.325475 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0331 04:05:54.341529 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:54.341554 3792 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:54.361521 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:05:54.362643 3792 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:05:55.240452 3792 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:05:55.240515 3792 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:05:55.266876 3792 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:55.266895 3792 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:05:56.272888 3792 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0331 04:06:06.564244 3792 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:06.564844 3792 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:07.415636 3792 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:07.415668 3792 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-btztn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:07.420956 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-btztn I0331 04:06:07.420980 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-btztn I0331 04:07:03.461600 3792 obj_retry.go:464] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:03.462195 3792 obj_retry.go:464] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:03.731972 3792 obj_retry.go:458] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:03.732026 3792 obj_retry.go:458] Detected object konflux-ui/proxy-646d664c7b-8jppf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:03.738350 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-646d664c7b-8jppf I0331 04:07:03.738377 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-646d664c7b-8jppf I0331 04:07:11.861641 3792 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:11.862206 3792 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:11.878296 3792 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:11.878320 3792 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:11.886701 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0331 04:07:11.886733 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0331 04:07:40.222619 3792 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:40.223272 3792 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:40.943199 3792 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:40.943220 3792 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:40.948578 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb I0331 04:07:40.948604 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-5srkb I0331 04:07:44.659123 3792 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0331 04:07:47.698115 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0331 04:07:53.209348 3792 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:53.209893 3792 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:53.961632 3792 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:53.961654 3792 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:53.967106 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx I0331 04:07:53.967134 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-tcdcx I0331 04:08:18.686947 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0331 04:08:59.567951 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=174 I0331 04:09:00.566536 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0331 04:09:08.680919 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0331 04:09:41.574343 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=64 I0331 04:10:03.859743 3792 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:03.860401 3792 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:04.727412 3792 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:04.727456 3792 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-nklzq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:06.705796 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0331 04:10:23.582701 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=421 I0331 04:11:59.668811 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0331 04:12:40.573613 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=18 I0331 04:12:45.685909 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0331 04:13:08.676502 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0331 04:13:33.692957 3792 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0331 04:13:48.700665 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0331 04:13:57.566936 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=71 I0331 04:13:57.662594 3792 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0331 04:14:49.045932 3792 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:49.045981 3792 obj_retry.go:458] Detected object tsf-tpa/create-db-p8489 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:49.050338 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-p8489 I0331 04:14:49.050368 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-p8489 I0331 04:14:50.658647 3792 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0331 04:14:54.030825 3792 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:54.030851 3792 obj_retry.go:458] Detected object tsf-tpa/migrate-db-q4jqv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:54.033776 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-q4jqv I0331 04:14:54.033799 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-q4jqv I0331 04:15:00.028641 3792 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:00.028693 3792 obj_retry.go:458] Detected object tsf-tpa/create-importers-n6cv9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:00.032625 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-n6cv9 I0331 04:15:00.032654 3792 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-n6cv9 I0331 04:16:07.585833 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=12 I0331 04:16:08.579124 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0331 04:16:17.569866 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0331 04:16:33.689671 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0331 04:16:59.683322 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0331 04:17:32.571056 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=9 I0331 04:18:28.708669 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0331 04:18:36.696595 3792 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0331 04:18:37.671382 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0331 04:19:06.576033 3792 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0331 04:19:41.688776 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0331 04:19:50.703305 3792 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0331 04:20:05.118750 3792 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:05.119348 3792 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:06.139579 3792 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:06.139626 3792 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cl6dj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0331 04:20:08.976561 3792 duration.go:461] Config duration recorder: measurement expired for pod/kube-system/ecr-credential-refresh-29582160-x57cm